Refactor pre-SSE4 versions of QS8/QC8 GEMM/IGEMM microkernels
- Replace sign-extension from 8 to 16 bits with a more efficient sequence
- Replace casts to uintptr_t to casts to typed pointer types where possible
PiperOrigin-RevId: 382654408
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c
index baf86a3..3d546ec 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -699,10 +699,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -730,10 +730,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1074,7 +1074,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c
index 2346da9..0831988 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c
@@ -162,9 +162,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -550,10 +550,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -578,7 +578,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -740,7 +740,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c
index 41a84cc..9b4fcf5 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -172,10 +172,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -193,10 +193,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -214,10 +214,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -235,10 +235,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -256,10 +256,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -277,10 +277,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -298,10 +298,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -319,10 +319,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -340,10 +340,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -361,10 +361,10 @@
const __m128i vk9x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i9 += 16;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -382,10 +382,10 @@
const __m128i vk10x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i10 += 16;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -403,10 +403,10 @@
const __m128i vk11x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i11 += 16;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -424,10 +424,10 @@
const __m128i vk12x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 200 * sizeof(int8_t)));
i12 += 16;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -445,10 +445,10 @@
const __m128i vk13x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 216 * sizeof(int8_t)));
i13 += 16;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -466,10 +466,10 @@
const __m128i vk14x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i14 += 16;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -487,10 +487,10 @@
const __m128i vk15x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 248 * sizeof(int8_t)));
i15 += 16;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -508,10 +508,10 @@
const __m128i vk16x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 264 * sizeof(int8_t)));
i16 += 16;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -529,10 +529,10 @@
const __m128i vk17x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i17 += 16;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -550,10 +550,10 @@
const __m128i vk18x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 296 * sizeof(int8_t)));
i18 += 16;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -571,10 +571,10 @@
const __m128i vk19x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 312 * sizeof(int8_t)));
i19 += 16;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -592,10 +592,10 @@
const __m128i vk20x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i20 += 16;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -613,10 +613,10 @@
const __m128i vk21x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 344 * sizeof(int8_t)));
i21 += 16;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -634,10 +634,10 @@
const __m128i vk22x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 360 * sizeof(int8_t)));
i22 += 16;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -655,10 +655,10 @@
const __m128i vk23x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i23 += 16;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -676,10 +676,10 @@
const __m128i vk24x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 392 * sizeof(int8_t)));
i24 += 16;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -699,10 +699,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -732,18 +732,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -755,8 +755,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -768,8 +768,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -781,8 +781,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -794,8 +794,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -807,8 +807,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -820,8 +820,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -833,8 +833,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -846,8 +846,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -859,8 +859,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -872,8 +872,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 160));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -885,8 +885,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 176));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -898,8 +898,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -911,8 +911,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 208));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -924,8 +924,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 224));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -937,8 +937,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -950,8 +950,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 256));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -963,8 +963,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 272));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -976,8 +976,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -989,8 +989,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 304));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -1002,8 +1002,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 320));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1015,8 +1015,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1028,8 +1028,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 352));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1041,8 +1041,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 368));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1054,8 +1054,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1076,7 +1076,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c
index 14e9bfc..9d8dc4d 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -699,10 +699,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -730,10 +730,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1074,7 +1074,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c
index cc3ecf7..06adbad 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c
@@ -162,9 +162,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -550,10 +550,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -578,7 +578,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -740,7 +740,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c
index a2cb3a2..198e8cf 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c
@@ -167,9 +167,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -555,10 +555,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -583,7 +583,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -745,7 +745,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c
index fade7eb..382842f 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -283,10 +283,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -314,10 +314,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -450,7 +450,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c
index 65ac968..a925cd2 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c
@@ -82,9 +82,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -230,10 +230,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -258,7 +258,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -324,7 +324,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c
index c585297..3462cc6 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -92,10 +92,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -113,10 +113,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -134,10 +134,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -155,10 +155,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -176,10 +176,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -197,10 +197,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -218,10 +218,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -239,10 +239,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -260,10 +260,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -283,10 +283,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -316,18 +316,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -339,8 +339,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -352,8 +352,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -365,8 +365,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -378,8 +378,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -391,8 +391,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -404,8 +404,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -417,8 +417,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -430,8 +430,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -452,7 +452,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c
index e5511f6..c6d38a3 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -283,10 +283,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -314,10 +314,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -450,7 +450,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c
index 883f7ad..dd52855 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c
@@ -82,9 +82,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -230,10 +230,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -258,7 +258,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -324,7 +324,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c
index 2f52e90..5fd55e7 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c
@@ -87,9 +87,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -235,10 +235,10 @@
__m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 16 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ w = (const void*) ((const float*) w + 16);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -263,7 +263,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -329,7 +329,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c b/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c
index fca3f13..773b74b 100644
--- a/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c
+++ b/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -903,12 +903,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -945,10 +945,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1289,7 +1289,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c b/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c
index 8472af0..96d8444 100644
--- a/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c
+++ b/src/qc8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c
@@ -162,11 +162,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -704,12 +704,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -743,7 +743,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -905,7 +905,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c b/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c
index f6b06bd..8de3a3c 100644
--- a/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c
+++ b/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -176,12 +176,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -205,12 +205,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -234,12 +234,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -263,12 +263,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -292,12 +292,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -321,12 +321,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -350,12 +350,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -379,12 +379,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -408,12 +408,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -437,12 +437,12 @@
const __m128i vk9xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i9 += 24;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
- const __m128i vxi9xGHIJKLMN = _mm_unpacklo_epi8(vi9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9xGHIJKLMN));
- const __m128i vxk9xGHIJKLMN = _mm_unpacklo_epi8(vk9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9xGHIJKLMN));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
+ const __m128i vxi9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi9xGHIJKLMN, vi9xGHIJKLMN), 8);
+ const __m128i vxk9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk9xGHIJKLMN, vk9xGHIJKLMN), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -466,12 +466,12 @@
const __m128i vk10xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 256 * sizeof(int8_t)));
i10 += 24;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
- const __m128i vxi10xGHIJKLMN = _mm_unpacklo_epi8(vi10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10xGHIJKLMN));
- const __m128i vxk10xGHIJKLMN = _mm_unpacklo_epi8(vk10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10xGHIJKLMN));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
+ const __m128i vxi10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi10xGHIJKLMN, vi10xGHIJKLMN), 8);
+ const __m128i vxk10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk10xGHIJKLMN, vk10xGHIJKLMN), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -495,12 +495,12 @@
const __m128i vk11xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i11 += 24;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
- const __m128i vxi11xGHIJKLMN = _mm_unpacklo_epi8(vi11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11xGHIJKLMN));
- const __m128i vxk11xGHIJKLMN = _mm_unpacklo_epi8(vk11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11xGHIJKLMN));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
+ const __m128i vxi11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi11xGHIJKLMN, vi11xGHIJKLMN), 8);
+ const __m128i vxk11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk11xGHIJKLMN, vk11xGHIJKLMN), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -524,12 +524,12 @@
const __m128i vk12xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 304 * sizeof(int8_t)));
i12 += 24;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
- const __m128i vxi12xGHIJKLMN = _mm_unpacklo_epi8(vi12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12xGHIJKLMN));
- const __m128i vxk12xGHIJKLMN = _mm_unpacklo_epi8(vk12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12xGHIJKLMN));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
+ const __m128i vxi12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi12xGHIJKLMN, vi12xGHIJKLMN), 8);
+ const __m128i vxk12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk12xGHIJKLMN, vk12xGHIJKLMN), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -553,12 +553,12 @@
const __m128i vk13xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i13 += 24;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
- const __m128i vxi13xGHIJKLMN = _mm_unpacklo_epi8(vi13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13xGHIJKLMN));
- const __m128i vxk13xGHIJKLMN = _mm_unpacklo_epi8(vk13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13xGHIJKLMN));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
+ const __m128i vxi13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi13xGHIJKLMN, vi13xGHIJKLMN), 8);
+ const __m128i vxk13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk13xGHIJKLMN, vk13xGHIJKLMN), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -582,12 +582,12 @@
const __m128i vk14xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 352 * sizeof(int8_t)));
i14 += 24;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
- const __m128i vxi14xGHIJKLMN = _mm_unpacklo_epi8(vi14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14xGHIJKLMN));
- const __m128i vxk14xGHIJKLMN = _mm_unpacklo_epi8(vk14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14xGHIJKLMN));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
+ const __m128i vxi14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi14xGHIJKLMN, vi14xGHIJKLMN), 8);
+ const __m128i vxk14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk14xGHIJKLMN, vk14xGHIJKLMN), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -611,12 +611,12 @@
const __m128i vk15xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i15 += 24;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
- const __m128i vxi15xGHIJKLMN = _mm_unpacklo_epi8(vi15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15xGHIJKLMN));
- const __m128i vxk15xGHIJKLMN = _mm_unpacklo_epi8(vk15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15xGHIJKLMN));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
+ const __m128i vxi15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi15xGHIJKLMN, vi15xGHIJKLMN), 8);
+ const __m128i vxk15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk15xGHIJKLMN, vk15xGHIJKLMN), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -640,12 +640,12 @@
const __m128i vk16xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 400 * sizeof(int8_t)));
i16 += 24;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
- const __m128i vxi16xGHIJKLMN = _mm_unpacklo_epi8(vi16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16xGHIJKLMN));
- const __m128i vxk16xGHIJKLMN = _mm_unpacklo_epi8(vk16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16xGHIJKLMN));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
+ const __m128i vxi16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi16xGHIJKLMN, vi16xGHIJKLMN), 8);
+ const __m128i vxk16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk16xGHIJKLMN, vk16xGHIJKLMN), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -669,12 +669,12 @@
const __m128i vk17xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 424 * sizeof(int8_t)));
i17 += 24;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
- const __m128i vxi17xGHIJKLMN = _mm_unpacklo_epi8(vi17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17xGHIJKLMN));
- const __m128i vxk17xGHIJKLMN = _mm_unpacklo_epi8(vk17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17xGHIJKLMN));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
+ const __m128i vxi17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi17xGHIJKLMN, vi17xGHIJKLMN), 8);
+ const __m128i vxk17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk17xGHIJKLMN, vk17xGHIJKLMN), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -698,12 +698,12 @@
const __m128i vk18xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 448 * sizeof(int8_t)));
i18 += 24;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
- const __m128i vxi18xGHIJKLMN = _mm_unpacklo_epi8(vi18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18xGHIJKLMN));
- const __m128i vxk18xGHIJKLMN = _mm_unpacklo_epi8(vk18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18xGHIJKLMN));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
+ const __m128i vxi18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi18xGHIJKLMN, vi18xGHIJKLMN), 8);
+ const __m128i vxk18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk18xGHIJKLMN, vk18xGHIJKLMN), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -727,12 +727,12 @@
const __m128i vk19xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 472 * sizeof(int8_t)));
i19 += 24;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
- const __m128i vxi19xGHIJKLMN = _mm_unpacklo_epi8(vi19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19xGHIJKLMN));
- const __m128i vxk19xGHIJKLMN = _mm_unpacklo_epi8(vk19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19xGHIJKLMN));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
+ const __m128i vxi19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi19xGHIJKLMN, vi19xGHIJKLMN), 8);
+ const __m128i vxk19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk19xGHIJKLMN, vk19xGHIJKLMN), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -756,12 +756,12 @@
const __m128i vk20xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 496 * sizeof(int8_t)));
i20 += 24;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
- const __m128i vxi20xGHIJKLMN = _mm_unpacklo_epi8(vi20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20xGHIJKLMN));
- const __m128i vxk20xGHIJKLMN = _mm_unpacklo_epi8(vk20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20xGHIJKLMN));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
+ const __m128i vxi20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi20xGHIJKLMN, vi20xGHIJKLMN), 8);
+ const __m128i vxk20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk20xGHIJKLMN, vk20xGHIJKLMN), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -785,12 +785,12 @@
const __m128i vk21xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 520 * sizeof(int8_t)));
i21 += 24;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
- const __m128i vxi21xGHIJKLMN = _mm_unpacklo_epi8(vi21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21xGHIJKLMN));
- const __m128i vxk21xGHIJKLMN = _mm_unpacklo_epi8(vk21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21xGHIJKLMN));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
+ const __m128i vxi21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi21xGHIJKLMN, vi21xGHIJKLMN), 8);
+ const __m128i vxk21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk21xGHIJKLMN, vk21xGHIJKLMN), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -814,12 +814,12 @@
const __m128i vk22xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 544 * sizeof(int8_t)));
i22 += 24;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
- const __m128i vxi22xGHIJKLMN = _mm_unpacklo_epi8(vi22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22xGHIJKLMN));
- const __m128i vxk22xGHIJKLMN = _mm_unpacklo_epi8(vk22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22xGHIJKLMN));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
+ const __m128i vxi22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi22xGHIJKLMN, vi22xGHIJKLMN), 8);
+ const __m128i vxk22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk22xGHIJKLMN, vk22xGHIJKLMN), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -843,12 +843,12 @@
const __m128i vk23xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 568 * sizeof(int8_t)));
i23 += 24;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
- const __m128i vxi23xGHIJKLMN = _mm_unpacklo_epi8(vi23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23xGHIJKLMN));
- const __m128i vxk23xGHIJKLMN = _mm_unpacklo_epi8(vk23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23xGHIJKLMN));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
+ const __m128i vxi23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi23xGHIJKLMN, vi23xGHIJKLMN), 8);
+ const __m128i vxk23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk23xGHIJKLMN, vk23xGHIJKLMN), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -872,12 +872,12 @@
const __m128i vk24xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 592 * sizeof(int8_t)));
i24 += 24;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
- const __m128i vxi24xGHIJKLMN = _mm_unpacklo_epi8(vi24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24xGHIJKLMN));
- const __m128i vxk24xGHIJKLMN = _mm_unpacklo_epi8(vk24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24xGHIJKLMN));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
+ const __m128i vxi24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi24xGHIJKLMN, vi24xGHIJKLMN), 8);
+ const __m128i vxk24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk24xGHIJKLMN, vk24xGHIJKLMN), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -903,12 +903,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -947,18 +947,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -970,8 +970,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -983,8 +983,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -996,8 +996,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -1009,8 +1009,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -1022,8 +1022,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -1035,8 +1035,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -1048,8 +1048,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -1061,8 +1061,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -1074,8 +1074,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 216));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -1087,8 +1087,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -1100,8 +1100,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 264));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -1113,8 +1113,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -1126,8 +1126,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 312));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -1139,8 +1139,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -1152,8 +1152,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 360));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -1165,8 +1165,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -1178,8 +1178,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 408));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -1191,8 +1191,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 432));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -1204,8 +1204,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 456));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -1217,8 +1217,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 480));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1230,8 +1230,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 504));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1243,8 +1243,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 528));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1256,8 +1256,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 552));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1269,8 +1269,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 576));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1291,7 +1291,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c b/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c
index 017f7ff..b922d02 100644
--- a/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c
+++ b/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -903,12 +903,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -945,10 +945,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1289,7 +1289,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c b/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c
index 2ca78b5..f505a66 100644
--- a/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c
+++ b/src/qc8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c
@@ -162,11 +162,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -704,12 +704,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -743,7 +743,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -905,7 +905,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c b/src/qc8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c
index 1aad934..34a245c 100644
--- a/src/qc8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c
+++ b/src/qc8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c
@@ -167,11 +167,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -709,12 +709,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -748,7 +748,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -910,7 +910,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c b/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c
index cbb3383..21f1cc9 100644
--- a/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c
+++ b/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -359,12 +359,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -401,10 +401,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -537,7 +537,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c b/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c
index d3af8e9..ede239d 100644
--- a/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c
+++ b/src/qc8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c
@@ -82,11 +82,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -288,12 +288,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -327,7 +327,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -393,7 +393,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c b/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c
index 3e08733..91be669 100644
--- a/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c
+++ b/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -96,12 +96,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -125,12 +125,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -154,12 +154,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -183,12 +183,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -212,12 +212,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -241,12 +241,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -270,12 +270,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -299,12 +299,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -328,12 +328,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -359,12 +359,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -403,18 +403,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -426,8 +426,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -439,8 +439,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -452,8 +452,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -465,8 +465,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -478,8 +478,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -491,8 +491,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -504,8 +504,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -517,8 +517,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -539,7 +539,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c b/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c
index 34367e5..56cef26 100644
--- a/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c
+++ b/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -359,12 +359,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -401,10 +401,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -537,7 +537,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c b/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c
index b890fd6..b8c8949 100644
--- a/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c
+++ b/src/qc8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c
@@ -82,11 +82,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -288,12 +288,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -327,7 +327,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -393,7 +393,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c b/src/qc8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c
index fc7eabe..51a24df 100644
--- a/src/qc8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c
+++ b/src/qc8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c
@@ -87,11 +87,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -293,12 +293,12 @@
__m128 vscaledKLMN = _mm_cvtepi32_ps(vaccKLMN);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- const __m128 vscale89AB = _mm_loadu_ps((const float*) ((uintptr_t) w + 8 * sizeof(float)));
- const __m128 vscaleCDEF = _mm_loadu_ps((const float*) ((uintptr_t) w + 12 * sizeof(float)));
- const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) ((uintptr_t) w + 16 * sizeof(float)));
- const __m128 vscaleKLMN = _mm_loadu_ps((const float*) ((uintptr_t) w + 20 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 24 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ const __m128 vscale89AB = _mm_loadu_ps((const float*) w + 8);
+ const __m128 vscaleCDEF = _mm_loadu_ps((const float*) w + 12);
+ const __m128 vscaleGHIJ = _mm_loadu_ps((const float*) w + 16);
+ const __m128 vscaleKLMN = _mm_loadu_ps((const float*) w + 20);
+ w = (const void*) ((const float*) w + 24);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
vscaled89AB = _mm_mul_ps(vscaled89AB, vscale89AB);
@@ -332,7 +332,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -398,7 +398,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c
index 769ee10..45e9fd8 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c
@@ -161,7 +161,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -495,8 +495,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -521,7 +521,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c
index 6817fe6..566f392 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c
@@ -162,7 +162,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -396,8 +396,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -417,7 +417,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -579,7 +579,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c
index 97cd2c4..5f5b0f4 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c
@@ -161,15 +161,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -181,8 +181,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -194,8 +194,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -207,8 +207,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -220,8 +220,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -233,8 +233,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -246,8 +246,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -259,8 +259,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -272,8 +272,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -285,8 +285,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -298,8 +298,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -311,8 +311,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -324,8 +324,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -337,8 +337,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -350,8 +350,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -363,8 +363,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -376,8 +376,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -389,8 +389,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -402,8 +402,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -415,8 +415,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -428,8 +428,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -441,8 +441,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -454,8 +454,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -467,8 +467,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -480,8 +480,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -495,8 +495,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -521,14 +521,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -539,8 +539,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -551,8 +551,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -563,8 +563,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -575,8 +575,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -587,8 +587,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -599,8 +599,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -611,8 +611,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -623,8 +623,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -635,8 +635,8 @@
const __m128i vi9x01234567 = _mm_loadl_epi64((const __m128i*) i9);
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -647,8 +647,8 @@
const __m128i vi10x01234567 = _mm_loadl_epi64((const __m128i*) i10);
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -659,8 +659,8 @@
const __m128i vi11x01234567 = _mm_loadl_epi64((const __m128i*) i11);
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -671,8 +671,8 @@
const __m128i vi12x01234567 = _mm_loadl_epi64((const __m128i*) i12);
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -683,8 +683,8 @@
const __m128i vi13x01234567 = _mm_loadl_epi64((const __m128i*) i13);
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -695,8 +695,8 @@
const __m128i vi14x01234567 = _mm_loadl_epi64((const __m128i*) i14);
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -707,8 +707,8 @@
const __m128i vi15x01234567 = _mm_loadl_epi64((const __m128i*) i15);
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -719,8 +719,8 @@
const __m128i vi16x01234567 = _mm_loadl_epi64((const __m128i*) i16);
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -731,8 +731,8 @@
const __m128i vi17x01234567 = _mm_loadl_epi64((const __m128i*) i17);
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -743,8 +743,8 @@
const __m128i vi18x01234567 = _mm_loadl_epi64((const __m128i*) i18);
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -755,8 +755,8 @@
const __m128i vi19x01234567 = _mm_loadl_epi64((const __m128i*) i19);
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -767,8 +767,8 @@
const __m128i vi20x01234567 = _mm_loadl_epi64((const __m128i*) i20);
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -779,8 +779,8 @@
const __m128i vi21x01234567 = _mm_loadl_epi64((const __m128i*) i21);
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -791,8 +791,8 @@
const __m128i vi22x01234567 = _mm_loadl_epi64((const __m128i*) i22);
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -803,8 +803,8 @@
const __m128i vi23x01234567 = _mm_loadl_epi64((const __m128i*) i23);
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -815,8 +815,8 @@
const __m128i vi24x01234567 = _mm_loadl_epi64((const __m128i*) i24);
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c
index e414aa9..d2c1e1c 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c
@@ -161,7 +161,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -495,8 +495,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -521,7 +521,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c
index 217457e..2833bf0 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c
@@ -162,7 +162,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -396,8 +396,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -417,7 +417,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -579,7 +579,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c
index 7a777b1..2a154b8 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c
@@ -167,7 +167,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -401,8 +401,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -422,7 +422,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -584,7 +584,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c
index 68cc238..22698b5 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c
@@ -81,7 +81,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -207,8 +207,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -233,7 +233,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c
index 6ee606b..1ae035f 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c
@@ -82,7 +82,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -172,8 +172,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -193,7 +193,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -259,7 +259,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c
index ee805d0..de64f1f 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c
@@ -81,15 +81,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -101,8 +101,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -114,8 +114,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -127,8 +127,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -140,8 +140,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -153,8 +153,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -166,8 +166,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -179,8 +179,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -192,8 +192,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -207,8 +207,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -233,14 +233,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -251,8 +251,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -263,8 +263,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -275,8 +275,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -287,8 +287,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -299,8 +299,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -311,8 +311,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -323,8 +323,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -335,8 +335,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c
index 9c1246f..4048532 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c
@@ -81,7 +81,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -207,8 +207,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -233,7 +233,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c
index e705682..24e07eb 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c
@@ -82,7 +82,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -172,8 +172,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -193,7 +193,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -259,7 +259,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c
index 4b02baa..66d660b 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c
@@ -87,7 +87,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -177,8 +177,8 @@
__m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- const __m128 vscale4567 = _mm_loadu_ps((const float*) ((uintptr_t) w + 4 * sizeof(float)));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(float));
+ const __m128 vscale4567 = _mm_loadu_ps((const float*) w + 4);
+ w = (const void*) ((const float*) w + 8);
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vscaled4567 = _mm_mul_ps(vscaled4567, vscale4567);
@@ -198,7 +198,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -264,7 +264,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, vscale0123);
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
index f2e24a7..a7e330c 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,7 +72,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -109,7 +109,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
index 2a04fc8..1e364fd 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -56,23 +56,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -109,7 +109,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
index 617fc79..5239bde 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -43,12 +43,12 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,33 +72,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -109,7 +109,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
index ddc1095..8a59055 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -43,62 +43,62 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -109,7 +109,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
index e213352..9dcb26f 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,7 +72,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -109,7 +109,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
index 048febc..1a60399 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -56,23 +56,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -109,7 +109,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
index 6b65214..9ac6301 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -48,7 +48,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -66,7 +66,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -77,7 +77,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -87,7 +87,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -95,7 +95,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -103,7 +103,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -114,7 +114,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
index a182e89..6ee7bf9 100644
--- a/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -48,7 +48,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,23 +61,23 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -87,7 +87,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -95,7 +95,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -103,7 +103,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -114,7 +114,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
index 82f7100..d9c1d2c 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -80,7 +80,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
index 709321b..3235071 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -57,20 +57,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -82,7 +82,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
index f82da93..2cad520 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -45,12 +45,12 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -80,7 +80,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
index dceacec..d595a83 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -45,32 +45,32 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -82,7 +82,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
index 4b0bffd..57e2d8a 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -80,7 +80,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
index 0c1870e..7e33913 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -57,20 +57,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -82,7 +82,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
index 239f15d..3de92c0 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -50,7 +50,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -65,7 +65,7 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -73,7 +73,7 @@
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -85,7 +85,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
index 9acbca1..8c3f029 100644
--- a/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qc8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -50,7 +50,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -62,20 +62,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -87,7 +87,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
index b208c3b..865a606 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -137,7 +137,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
index 701c66d..fcf7354 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -68,21 +68,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -137,7 +137,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
index f232688..d539fca 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -50,15 +50,15 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,20 +90,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -137,7 +137,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
index cbfea83..7190ea9 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -50,60 +50,60 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -137,7 +137,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
index 8ace15a..8ac0f3e 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -137,7 +137,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
index f1c07c1..6e0bdea 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -68,21 +68,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -137,7 +137,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
index 7dcb9d9..e9b1522 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -95,7 +95,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -108,7 +108,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -118,7 +118,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -128,7 +128,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -142,7 +142,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
index ea2a328..0f6a85f 100644
--- a/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -73,21 +73,21 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -95,7 +95,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -108,7 +108,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -118,7 +118,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -128,7 +128,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -142,7 +142,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
index 3595651..8246e47 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -101,7 +101,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
index d9b0072..9e36562 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -71,23 +71,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -103,7 +103,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
index 7ab73a6..abc9ae3 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -55,15 +55,15 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -101,7 +101,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
index e205ebe..2818fd5 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -55,39 +55,39 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -103,7 +103,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
index 7fa45b8..ca0ee71 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -101,7 +101,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
index d758e6c..5ef3176 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -71,23 +71,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -103,7 +103,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
index 33313e4..9506d4c 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -80,7 +80,7 @@
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -106,7 +106,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
index 7d2cffe..f204d5a 100644
--- a/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qc8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -76,23 +76,23 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -108,7 +108,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
index 1550ecc..00e88af 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -165,7 +165,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
index bf95edb..b61b9da 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -98,7 +98,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -165,7 +165,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
index e337bbc..eef063f 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -57,18 +57,18 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -165,7 +165,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
index bb1730c..c17ae1f 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -57,22 +57,22 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -80,8 +80,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -89,8 +89,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -98,8 +98,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -165,7 +165,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
index f5f3f1c..042edf5 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -165,7 +165,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
index 042d7b5..e193a29 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -98,7 +98,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -165,7 +165,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
index def9e5e..aa84b24 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -62,7 +62,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -94,7 +94,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -113,7 +113,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -129,7 +129,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -141,7 +141,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -153,7 +153,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -170,7 +170,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
index 184754d..0fe3848 100644
--- a/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -62,7 +62,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -85,7 +85,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -94,7 +94,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -103,7 +103,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -113,7 +113,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -129,7 +129,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -141,7 +141,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -153,7 +153,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -170,7 +170,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
index 5744d2a..c637a2b 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -122,7 +122,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
index a6246de..98dd5ae 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -85,26 +85,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -124,7 +124,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
index 8275917..53df1af 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -65,18 +65,18 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -122,7 +122,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
index 80d112b..11e8f69 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -65,46 +65,46 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -124,7 +124,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
index b1209ec..a6baeff 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -122,7 +122,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
index 6d9a6bd..3638c0b 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -85,26 +85,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -124,7 +124,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
index 904263f..7c52bd7 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -70,7 +70,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -95,7 +95,7 @@
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -107,7 +107,7 @@
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -127,7 +127,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
index 48bec57..9f03b0f 100644
--- a/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qc8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -70,7 +70,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,26 +90,26 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -129,7 +129,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
index e18d5d7..0ac2672 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -193,7 +193,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
index d6f5fc2..109c96e 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -92,7 +92,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -114,7 +114,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -193,7 +193,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
index 0d9da78..a697ab2 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -64,21 +64,21 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -193,7 +193,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
index 81f3f5c..2f1385f 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -64,25 +64,25 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -92,8 +92,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -114,8 +114,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -193,7 +193,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
index f9b111d..4af3657 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -193,7 +193,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
index 965ef36..3fb83bb 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -92,7 +92,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -114,7 +114,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -193,7 +193,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
index f0b662e..425cb41 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -69,7 +69,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -131,7 +131,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -150,7 +150,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -164,7 +164,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -178,7 +178,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -198,7 +198,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
index ca51f86..50da9d5 100644
--- a/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -69,7 +69,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -97,7 +97,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -119,7 +119,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -131,7 +131,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -150,7 +150,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -164,7 +164,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -178,7 +178,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -198,7 +198,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
index 578845e..37e5e14 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,7 +82,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -121,7 +121,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
index e05d50a..2922f31 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -66,23 +66,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -121,7 +121,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
index 5d1bab5..1c092df 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,7 +58,7 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,33 +82,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -121,7 +121,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
index 1bac56b..6d7e3db 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,57 +58,57 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -121,7 +121,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
index 9feb8ff..7c4b5d9 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,7 +82,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -121,7 +121,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
index 096cc50..bef8386 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -66,23 +66,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -121,7 +121,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
index 8563c33..68b4668 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,7 +76,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -87,7 +87,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -96,24 +96,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -126,7 +126,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
index 5f73380..ab20c98 100644
--- a/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,23 +71,23 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -96,24 +96,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -126,7 +126,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
index 11f4972..4f41a4a 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -93,7 +93,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
index ff80586..c886df4 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -68,20 +68,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -95,7 +95,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
index 4e8c01d..f08d4ad 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,7 +61,7 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -93,7 +93,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
index 7820121..bb4d21f 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,27 +61,27 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -95,7 +95,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
index 5200238..682f78c 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -93,7 +93,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
index 4376e65..98f80bf 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -68,20 +68,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -95,7 +95,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
index 3cbf83e..8ab761a 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -53,7 +53,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,7 +76,7 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -84,7 +84,7 @@
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -98,7 +98,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
index e028b42..fb13c69 100644
--- a/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qc8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -53,7 +53,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,20 +73,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
@@ -100,7 +100,7 @@
__m128 vscaled0x0123 = _mm_cvtepi32_ps(vacc0x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vacc0x0123 = _mm_cvtps_epi32(vscaled0x0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
index c24cae1..b563212 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -151,7 +151,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
index 84c1823..3d0d957 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -80,21 +80,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -151,7 +151,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
index 42c06c0..8b9932d 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,10 +67,10 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,20 +102,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -151,7 +151,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
index 1684f7f..248de43 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,55 +67,55 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -151,7 +151,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
index 1fda902..30a132d 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -151,7 +151,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
index 6d7af73..fbb67cd 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -80,21 +80,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -151,7 +151,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
index da7704e..a4d0108 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -92,7 +92,7 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -107,7 +107,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,8 +119,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -129,8 +129,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -139,8 +139,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -156,7 +156,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
index a8c6337..6faebfb 100644
--- a/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,21 +85,21 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -107,7 +107,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,8 +119,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -129,8 +129,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -139,8 +139,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -156,7 +156,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
index 0fe37e8..564303d 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -116,7 +116,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
index b5a2944..f620ac3 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -118,7 +118,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
index f16164d..0acc002 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,10 +73,10 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -116,7 +116,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
index 78e19f9..5be9dd9 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,34 +73,34 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -118,7 +118,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
index 1b6e3bc..b56c544 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -116,7 +116,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
index 49ce8d2..65b76cc 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -118,7 +118,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
index 8c36446..53ccc2e 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -61,7 +61,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -93,7 +93,7 @@
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -103,7 +103,7 @@
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -121,7 +121,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
index 046e703..a9724f0 100644
--- a/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qc8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -61,7 +61,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -89,23 +89,23 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
@@ -123,7 +123,7 @@
__m128 vscaled1x0123 = _mm_cvtepi32_ps(vacc1x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
index 4e873a6..adce9d5 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -181,7 +181,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
index 1210e2c..5ba6759 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -94,7 +94,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -112,7 +112,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -181,7 +181,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
index 387196c..03db5db 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,13 +76,13 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -181,7 +181,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
index 3da6524..9f01bff 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,17 +76,17 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -94,8 +94,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -112,8 +112,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -181,7 +181,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
index 3e9f7e0..c3ee33f 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -181,7 +181,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
index b060bfa..434f062 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -94,7 +94,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -112,7 +112,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -181,7 +181,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
index 06006b8..0d80e05 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -127,7 +127,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -142,8 +142,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -154,8 +154,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -166,8 +166,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -186,7 +186,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
index 807bef7..d71485e 100644
--- a/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -99,7 +99,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -117,7 +117,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -127,7 +127,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -142,8 +142,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -154,8 +154,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -166,8 +166,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -186,7 +186,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
index f4d21ea..fa28748 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -139,7 +139,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
index 8408090..156751a 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -100,26 +100,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -141,7 +141,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
index 1e7423e..4e5ad2a 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,13 +85,13 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -139,7 +139,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
index c84c7cd..547507a 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,41 +85,41 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -141,7 +141,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
index 39f2ca5..9b5b688 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -139,7 +139,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
index 6a4b9d4..b786576 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -100,26 +100,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -141,7 +141,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
index 4369f6d..edcbe65 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -69,7 +69,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -110,7 +110,7 @@
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -144,7 +144,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
index 0433e8f..00c7c77 100644
--- a/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qc8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -69,7 +69,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,26 +105,26 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
@@ -146,7 +146,7 @@
__m128 vscaled2x0123 = _mm_cvtepi32_ps(vacc2x0123);
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
index 9118bd1..afef149 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -211,7 +211,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
index affc814..715a6f4 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -130,7 +130,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -211,7 +211,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
index cb05a57..4a33891 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,16 +85,16 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -211,7 +211,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
index 95bfccf..c6ec484 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,20 +85,20 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -108,8 +108,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -119,8 +119,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -130,8 +130,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -211,7 +211,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
index 0035013..559fa02 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -211,7 +211,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
index c4f0590..73c236c 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -130,7 +130,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -211,7 +211,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
index dcd4f0a..f29ac8e 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -124,7 +124,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -147,7 +147,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -165,8 +165,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -179,8 +179,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -193,8 +193,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -216,7 +216,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
index 5b38754..5678e70 100644
--- a/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qc8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -113,7 +113,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -124,7 +124,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -135,7 +135,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -147,7 +147,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -165,8 +165,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -179,8 +179,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -193,8 +193,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -216,7 +216,7 @@
__m128 vscaled3x0123 = _mm_cvtepi32_ps(vacc3x0123);
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
vscaled0x0123 = _mm_mul_ps(vscaled0x0123, vscale0123);
vscaled1x0123 = _mm_mul_ps(vscaled1x0123, vscale0123);
vscaled2x0123 = _mm_mul_ps(vscaled2x0123, vscale0123);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c
index 23dd794..c87a54c 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -726,10 +726,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1069,7 +1069,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c
index db874ce..6f6b7c2 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-avx-mul32.c
@@ -162,9 +162,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -574,7 +574,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -735,7 +735,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c
index 6027deb..8de2bd9 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse2-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -172,10 +172,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -193,10 +193,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -214,10 +214,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -235,10 +235,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -256,10 +256,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -277,10 +277,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -298,10 +298,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -319,10 +319,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -340,10 +340,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -361,10 +361,10 @@
const __m128i vk9x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i9 += 16;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -382,10 +382,10 @@
const __m128i vk10x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i10 += 16;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -403,10 +403,10 @@
const __m128i vk11x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i11 += 16;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -424,10 +424,10 @@
const __m128i vk12x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 200 * sizeof(int8_t)));
i12 += 16;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -445,10 +445,10 @@
const __m128i vk13x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 216 * sizeof(int8_t)));
i13 += 16;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -466,10 +466,10 @@
const __m128i vk14x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i14 += 16;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -487,10 +487,10 @@
const __m128i vk15x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 248 * sizeof(int8_t)));
i15 += 16;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -508,10 +508,10 @@
const __m128i vk16x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 264 * sizeof(int8_t)));
i16 += 16;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -529,10 +529,10 @@
const __m128i vk17x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i17 += 16;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -550,10 +550,10 @@
const __m128i vk18x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 296 * sizeof(int8_t)));
i18 += 16;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -571,10 +571,10 @@
const __m128i vk19x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 312 * sizeof(int8_t)));
i19 += 16;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -592,10 +592,10 @@
const __m128i vk20x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i20 += 16;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -613,10 +613,10 @@
const __m128i vk21x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 344 * sizeof(int8_t)));
i21 += 16;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -634,10 +634,10 @@
const __m128i vk22x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 360 * sizeof(int8_t)));
i22 += 16;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -655,10 +655,10 @@
const __m128i vk23x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i23 += 16;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -676,10 +676,10 @@
const __m128i vk24x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 392 * sizeof(int8_t)));
i24 += 16;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -728,18 +728,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -751,8 +751,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -764,8 +764,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -777,8 +777,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -790,8 +790,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -803,8 +803,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -816,8 +816,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -829,8 +829,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -842,8 +842,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -855,8 +855,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -868,8 +868,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 160));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -881,8 +881,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 176));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -894,8 +894,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -907,8 +907,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 208));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -920,8 +920,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 224));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -933,8 +933,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -946,8 +946,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 256));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -959,8 +959,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 272));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -972,8 +972,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -985,8 +985,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 304));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -998,8 +998,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 320));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1011,8 +1011,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1024,8 +1024,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 352));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1037,8 +1037,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 368));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1050,8 +1050,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1071,7 +1071,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c
index 5b16db5..47ed165 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -726,10 +726,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1069,7 +1069,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c
index e2de409..98652e9 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-sse41-mul32.c
@@ -162,9 +162,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -574,7 +574,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -735,7 +735,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c
index 5e58f82..3f44573 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-fp32-xop-mul32.c
@@ -167,9 +167,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -579,7 +579,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -740,7 +740,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c
index 1e05685..3fc1b06 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -761,10 +761,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1127,7 +1127,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c
index 39e5891..dbe7341 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-avx-mul32.c
@@ -162,9 +162,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -609,7 +609,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -788,7 +788,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c
index a13f7ff..bc2b79c 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse2-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -172,10 +172,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -193,10 +193,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -214,10 +214,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -235,10 +235,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -256,10 +256,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -277,10 +277,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -298,10 +298,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -319,10 +319,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -340,10 +340,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -361,10 +361,10 @@
const __m128i vk9x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i9 += 16;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -382,10 +382,10 @@
const __m128i vk10x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i10 += 16;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -403,10 +403,10 @@
const __m128i vk11x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i11 += 16;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -424,10 +424,10 @@
const __m128i vk12x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 200 * sizeof(int8_t)));
i12 += 16;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -445,10 +445,10 @@
const __m128i vk13x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 216 * sizeof(int8_t)));
i13 += 16;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -466,10 +466,10 @@
const __m128i vk14x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i14 += 16;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -487,10 +487,10 @@
const __m128i vk15x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 248 * sizeof(int8_t)));
i15 += 16;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -508,10 +508,10 @@
const __m128i vk16x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 264 * sizeof(int8_t)));
i16 += 16;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -529,10 +529,10 @@
const __m128i vk17x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i17 += 16;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -550,10 +550,10 @@
const __m128i vk18x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 296 * sizeof(int8_t)));
i18 += 16;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -571,10 +571,10 @@
const __m128i vk19x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 312 * sizeof(int8_t)));
i19 += 16;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -592,10 +592,10 @@
const __m128i vk20x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i20 += 16;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -613,10 +613,10 @@
const __m128i vk21x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 344 * sizeof(int8_t)));
i21 += 16;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -634,10 +634,10 @@
const __m128i vk22x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 360 * sizeof(int8_t)));
i22 += 16;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -655,10 +655,10 @@
const __m128i vk23x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i23 += 16;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -676,10 +676,10 @@
const __m128i vk24x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 392 * sizeof(int8_t)));
i24 += 16;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -800,18 +800,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -823,8 +823,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -836,8 +836,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -849,8 +849,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -862,8 +862,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -875,8 +875,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -888,8 +888,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -901,8 +901,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -914,8 +914,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -927,8 +927,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -940,8 +940,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 160));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -953,8 +953,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 176));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -966,8 +966,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -979,8 +979,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 208));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -992,8 +992,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 224));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -1005,8 +1005,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -1018,8 +1018,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 256));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -1031,8 +1031,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 272));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -1044,8 +1044,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -1057,8 +1057,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 304));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -1070,8 +1070,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 320));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1083,8 +1083,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1096,8 +1096,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 352));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1109,8 +1109,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 368));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1122,8 +1122,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1186,7 +1186,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c
index 9439589..0fcdc55 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -761,10 +761,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1127,7 +1127,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c
index 5838b80..1ea33c3 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-sse41-mul32.c
@@ -162,9 +162,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -609,7 +609,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -788,7 +788,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c
index b16af4a..3855261 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-ssse3-mul16.c
@@ -161,9 +161,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -172,10 +172,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -193,10 +193,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -214,10 +214,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -235,10 +235,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -256,10 +256,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -277,10 +277,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -298,10 +298,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -319,10 +319,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -340,10 +340,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -361,10 +361,10 @@
const __m128i vk9x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i9 += 16;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -382,10 +382,10 @@
const __m128i vk10x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i10 += 16;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -403,10 +403,10 @@
const __m128i vk11x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i11 += 16;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -424,10 +424,10 @@
const __m128i vk12x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 200 * sizeof(int8_t)));
i12 += 16;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -445,10 +445,10 @@
const __m128i vk13x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 216 * sizeof(int8_t)));
i13 += 16;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -466,10 +466,10 @@
const __m128i vk14x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i14 += 16;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -487,10 +487,10 @@
const __m128i vk15x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 248 * sizeof(int8_t)));
i15 += 16;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -508,10 +508,10 @@
const __m128i vk16x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 264 * sizeof(int8_t)));
i16 += 16;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -529,10 +529,10 @@
const __m128i vk17x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i17 += 16;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -550,10 +550,10 @@
const __m128i vk18x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 296 * sizeof(int8_t)));
i18 += 16;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -571,10 +571,10 @@
const __m128i vk19x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 312 * sizeof(int8_t)));
i19 += 16;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -592,10 +592,10 @@
const __m128i vk20x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i20 += 16;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -613,10 +613,10 @@
const __m128i vk21x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 344 * sizeof(int8_t)));
i21 += 16;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -634,10 +634,10 @@
const __m128i vk22x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 360 * sizeof(int8_t)));
i22 += 16;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -655,10 +655,10 @@
const __m128i vk23x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i23 += 16;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -676,10 +676,10 @@
const __m128i vk24x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 392 * sizeof(int8_t)));
i24 += 16;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -800,18 +800,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -823,8 +823,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -836,8 +836,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -849,8 +849,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -862,8 +862,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -875,8 +875,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -888,8 +888,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -901,8 +901,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -914,8 +914,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -927,8 +927,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -940,8 +940,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 160));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -953,8 +953,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 176));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -966,8 +966,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -979,8 +979,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 208));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -992,8 +992,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 224));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -1005,8 +1005,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -1018,8 +1018,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 256));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -1031,8 +1031,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 272));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -1044,8 +1044,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -1057,8 +1057,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 304));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -1070,8 +1070,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 320));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1083,8 +1083,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1096,8 +1096,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 352));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1109,8 +1109,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 368));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1122,8 +1122,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1186,7 +1186,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c
index 1dcb92f..5093cf0 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-xop-mul32.c
@@ -167,9 +167,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -614,7 +614,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -793,7 +793,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c
index 3e60603..58ff971 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -310,10 +310,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -445,7 +445,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c
index 3e9b092..d1592b6 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-avx-mul32.c
@@ -82,9 +82,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -254,7 +254,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -319,7 +319,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c
index 4b94f2b..6241c77 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse2-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -92,10 +92,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -113,10 +113,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -134,10 +134,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -155,10 +155,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -176,10 +176,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -197,10 +197,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -218,10 +218,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -239,10 +239,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -260,10 +260,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -312,18 +312,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -335,8 +335,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -348,8 +348,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -361,8 +361,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -374,8 +374,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -387,8 +387,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -400,8 +400,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -413,8 +413,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -426,8 +426,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -447,7 +447,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c
index 3f69930..8f2f88b 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -310,10 +310,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -445,7 +445,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c
index 53f52d1..ef07590 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-sse41-mul32.c
@@ -82,9 +82,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -254,7 +254,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -319,7 +319,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c
index 5a92646..35ff8f6 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-fp32-xop-mul32.c
@@ -87,9 +87,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -259,7 +259,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -324,7 +324,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c
index 803b485..fde0210 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -345,10 +345,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -503,7 +503,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c
index e425c26..34a8a20 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-avx-mul32.c
@@ -82,9 +82,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -289,7 +289,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -372,7 +372,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c
index 5a1e354..3ab57ed 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse2-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -92,10 +92,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -113,10 +113,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -134,10 +134,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -155,10 +155,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -176,10 +176,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -197,10 +197,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -218,10 +218,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -239,10 +239,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -260,10 +260,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -384,18 +384,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -407,8 +407,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -420,8 +420,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -433,8 +433,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -446,8 +446,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -459,8 +459,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -472,8 +472,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -485,8 +485,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -498,8 +498,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -562,7 +562,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c
index 81f2aeb..b3734a5 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -345,10 +345,10 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -503,7 +503,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c
index c23fd48..68533a2 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-sse41-mul32.c
@@ -82,9 +82,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -289,7 +289,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -372,7 +372,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c
index b9e2953..9b3679c 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-ssse3-mul16.c
@@ -81,9 +81,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -92,10 +92,10 @@
const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i0 += 16;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -113,10 +113,10 @@
const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i1 += 16;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -134,10 +134,10 @@
const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i2 += 16;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -155,10 +155,10 @@
const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i3 += 16;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -176,10 +176,10 @@
const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i4 += 16;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -197,10 +197,10 @@
const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i5 += 16;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -218,10 +218,10 @@
const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i6 += 16;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -239,10 +239,10 @@
const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i7 += 16;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -260,10 +260,10 @@
const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i8 += 16;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -384,18 +384,18 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -407,8 +407,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 16));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -420,8 +420,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 32));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -433,8 +433,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -446,8 +446,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 64));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -459,8 +459,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 80));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -472,8 +472,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -485,8 +485,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 112));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -498,8 +498,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 128));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -562,7 +562,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c
index f3eedf0..d3fe2ae 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-xop-mul32.c
@@ -87,9 +87,9 @@
const void* w = weights;
for (; c >= 16; c -= 16) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -294,7 +294,7 @@
output += 16;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 16 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -377,7 +377,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c
index 652f179..98bc938 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -939,10 +939,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1282,7 +1282,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c
index d0e7ac7..fa27362 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-avx-mul32.c
@@ -162,11 +162,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -737,7 +737,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -898,7 +898,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c
index 1a0f3f6..d351a2d 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse2-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -176,12 +176,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -205,12 +205,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -234,12 +234,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -263,12 +263,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -292,12 +292,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -321,12 +321,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -350,12 +350,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -379,12 +379,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -408,12 +408,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -437,12 +437,12 @@
const __m128i vk9xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i9 += 24;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
- const __m128i vxi9xGHIJKLMN = _mm_unpacklo_epi8(vi9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9xGHIJKLMN));
- const __m128i vxk9xGHIJKLMN = _mm_unpacklo_epi8(vk9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9xGHIJKLMN));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
+ const __m128i vxi9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi9xGHIJKLMN, vi9xGHIJKLMN), 8);
+ const __m128i vxk9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk9xGHIJKLMN, vk9xGHIJKLMN), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -466,12 +466,12 @@
const __m128i vk10xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 256 * sizeof(int8_t)));
i10 += 24;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
- const __m128i vxi10xGHIJKLMN = _mm_unpacklo_epi8(vi10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10xGHIJKLMN));
- const __m128i vxk10xGHIJKLMN = _mm_unpacklo_epi8(vk10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10xGHIJKLMN));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
+ const __m128i vxi10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi10xGHIJKLMN, vi10xGHIJKLMN), 8);
+ const __m128i vxk10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk10xGHIJKLMN, vk10xGHIJKLMN), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -495,12 +495,12 @@
const __m128i vk11xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i11 += 24;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
- const __m128i vxi11xGHIJKLMN = _mm_unpacklo_epi8(vi11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11xGHIJKLMN));
- const __m128i vxk11xGHIJKLMN = _mm_unpacklo_epi8(vk11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11xGHIJKLMN));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
+ const __m128i vxi11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi11xGHIJKLMN, vi11xGHIJKLMN), 8);
+ const __m128i vxk11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk11xGHIJKLMN, vk11xGHIJKLMN), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -524,12 +524,12 @@
const __m128i vk12xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 304 * sizeof(int8_t)));
i12 += 24;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
- const __m128i vxi12xGHIJKLMN = _mm_unpacklo_epi8(vi12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12xGHIJKLMN));
- const __m128i vxk12xGHIJKLMN = _mm_unpacklo_epi8(vk12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12xGHIJKLMN));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
+ const __m128i vxi12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi12xGHIJKLMN, vi12xGHIJKLMN), 8);
+ const __m128i vxk12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk12xGHIJKLMN, vk12xGHIJKLMN), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -553,12 +553,12 @@
const __m128i vk13xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i13 += 24;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
- const __m128i vxi13xGHIJKLMN = _mm_unpacklo_epi8(vi13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13xGHIJKLMN));
- const __m128i vxk13xGHIJKLMN = _mm_unpacklo_epi8(vk13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13xGHIJKLMN));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
+ const __m128i vxi13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi13xGHIJKLMN, vi13xGHIJKLMN), 8);
+ const __m128i vxk13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk13xGHIJKLMN, vk13xGHIJKLMN), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -582,12 +582,12 @@
const __m128i vk14xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 352 * sizeof(int8_t)));
i14 += 24;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
- const __m128i vxi14xGHIJKLMN = _mm_unpacklo_epi8(vi14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14xGHIJKLMN));
- const __m128i vxk14xGHIJKLMN = _mm_unpacklo_epi8(vk14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14xGHIJKLMN));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
+ const __m128i vxi14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi14xGHIJKLMN, vi14xGHIJKLMN), 8);
+ const __m128i vxk14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk14xGHIJKLMN, vk14xGHIJKLMN), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -611,12 +611,12 @@
const __m128i vk15xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i15 += 24;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
- const __m128i vxi15xGHIJKLMN = _mm_unpacklo_epi8(vi15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15xGHIJKLMN));
- const __m128i vxk15xGHIJKLMN = _mm_unpacklo_epi8(vk15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15xGHIJKLMN));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
+ const __m128i vxi15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi15xGHIJKLMN, vi15xGHIJKLMN), 8);
+ const __m128i vxk15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk15xGHIJKLMN, vk15xGHIJKLMN), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -640,12 +640,12 @@
const __m128i vk16xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 400 * sizeof(int8_t)));
i16 += 24;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
- const __m128i vxi16xGHIJKLMN = _mm_unpacklo_epi8(vi16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16xGHIJKLMN));
- const __m128i vxk16xGHIJKLMN = _mm_unpacklo_epi8(vk16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16xGHIJKLMN));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
+ const __m128i vxi16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi16xGHIJKLMN, vi16xGHIJKLMN), 8);
+ const __m128i vxk16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk16xGHIJKLMN, vk16xGHIJKLMN), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -669,12 +669,12 @@
const __m128i vk17xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 424 * sizeof(int8_t)));
i17 += 24;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
- const __m128i vxi17xGHIJKLMN = _mm_unpacklo_epi8(vi17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17xGHIJKLMN));
- const __m128i vxk17xGHIJKLMN = _mm_unpacklo_epi8(vk17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17xGHIJKLMN));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
+ const __m128i vxi17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi17xGHIJKLMN, vi17xGHIJKLMN), 8);
+ const __m128i vxk17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk17xGHIJKLMN, vk17xGHIJKLMN), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -698,12 +698,12 @@
const __m128i vk18xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 448 * sizeof(int8_t)));
i18 += 24;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
- const __m128i vxi18xGHIJKLMN = _mm_unpacklo_epi8(vi18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18xGHIJKLMN));
- const __m128i vxk18xGHIJKLMN = _mm_unpacklo_epi8(vk18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18xGHIJKLMN));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
+ const __m128i vxi18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi18xGHIJKLMN, vi18xGHIJKLMN), 8);
+ const __m128i vxk18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk18xGHIJKLMN, vk18xGHIJKLMN), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -727,12 +727,12 @@
const __m128i vk19xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 472 * sizeof(int8_t)));
i19 += 24;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
- const __m128i vxi19xGHIJKLMN = _mm_unpacklo_epi8(vi19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19xGHIJKLMN));
- const __m128i vxk19xGHIJKLMN = _mm_unpacklo_epi8(vk19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19xGHIJKLMN));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
+ const __m128i vxi19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi19xGHIJKLMN, vi19xGHIJKLMN), 8);
+ const __m128i vxk19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk19xGHIJKLMN, vk19xGHIJKLMN), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -756,12 +756,12 @@
const __m128i vk20xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 496 * sizeof(int8_t)));
i20 += 24;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
- const __m128i vxi20xGHIJKLMN = _mm_unpacklo_epi8(vi20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20xGHIJKLMN));
- const __m128i vxk20xGHIJKLMN = _mm_unpacklo_epi8(vk20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20xGHIJKLMN));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
+ const __m128i vxi20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi20xGHIJKLMN, vi20xGHIJKLMN), 8);
+ const __m128i vxk20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk20xGHIJKLMN, vk20xGHIJKLMN), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -785,12 +785,12 @@
const __m128i vk21xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 520 * sizeof(int8_t)));
i21 += 24;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
- const __m128i vxi21xGHIJKLMN = _mm_unpacklo_epi8(vi21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21xGHIJKLMN));
- const __m128i vxk21xGHIJKLMN = _mm_unpacklo_epi8(vk21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21xGHIJKLMN));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
+ const __m128i vxi21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi21xGHIJKLMN, vi21xGHIJKLMN), 8);
+ const __m128i vxk21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk21xGHIJKLMN, vk21xGHIJKLMN), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -814,12 +814,12 @@
const __m128i vk22xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 544 * sizeof(int8_t)));
i22 += 24;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
- const __m128i vxi22xGHIJKLMN = _mm_unpacklo_epi8(vi22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22xGHIJKLMN));
- const __m128i vxk22xGHIJKLMN = _mm_unpacklo_epi8(vk22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22xGHIJKLMN));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
+ const __m128i vxi22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi22xGHIJKLMN, vi22xGHIJKLMN), 8);
+ const __m128i vxk22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk22xGHIJKLMN, vk22xGHIJKLMN), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -843,12 +843,12 @@
const __m128i vk23xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 568 * sizeof(int8_t)));
i23 += 24;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
- const __m128i vxi23xGHIJKLMN = _mm_unpacklo_epi8(vi23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23xGHIJKLMN));
- const __m128i vxk23xGHIJKLMN = _mm_unpacklo_epi8(vk23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23xGHIJKLMN));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
+ const __m128i vxi23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi23xGHIJKLMN, vi23xGHIJKLMN), 8);
+ const __m128i vxk23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk23xGHIJKLMN, vk23xGHIJKLMN), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -872,12 +872,12 @@
const __m128i vk24xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 592 * sizeof(int8_t)));
i24 += 24;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
- const __m128i vxi24xGHIJKLMN = _mm_unpacklo_epi8(vi24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24xGHIJKLMN));
- const __m128i vxk24xGHIJKLMN = _mm_unpacklo_epi8(vk24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24xGHIJKLMN));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
+ const __m128i vxi24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi24xGHIJKLMN, vi24xGHIJKLMN), 8);
+ const __m128i vxk24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk24xGHIJKLMN, vk24xGHIJKLMN), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -941,18 +941,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -964,8 +964,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -977,8 +977,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -990,8 +990,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -1003,8 +1003,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -1016,8 +1016,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -1029,8 +1029,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -1042,8 +1042,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -1055,8 +1055,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -1068,8 +1068,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 216));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -1081,8 +1081,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -1094,8 +1094,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 264));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -1107,8 +1107,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -1120,8 +1120,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 312));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -1133,8 +1133,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -1146,8 +1146,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 360));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -1159,8 +1159,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -1172,8 +1172,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 408));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -1185,8 +1185,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 432));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -1198,8 +1198,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 456));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -1211,8 +1211,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 480));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1224,8 +1224,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 504));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1237,8 +1237,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 528));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1250,8 +1250,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 552));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1263,8 +1263,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 576));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1284,7 +1284,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c
index 6e299fc..5c180c7 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -939,10 +939,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1282,7 +1282,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c
index 7582306..105c54e 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-sse41-mul32.c
@@ -162,11 +162,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -737,7 +737,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -898,7 +898,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c
index faec623..476e4e0 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-fp32-xop-mul32.c
@@ -167,11 +167,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -742,7 +742,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -903,7 +903,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c
index 51e4c60..d394ad6 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -988,10 +988,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1354,7 +1354,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c
index e637cdd..4023ab2 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-avx-mul32.c
@@ -162,11 +162,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -786,7 +786,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -965,7 +965,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c
index 9b17ace..b5052f7 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse2-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -176,12 +176,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -205,12 +205,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -234,12 +234,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -263,12 +263,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -292,12 +292,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -321,12 +321,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -350,12 +350,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -379,12 +379,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -408,12 +408,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -437,12 +437,12 @@
const __m128i vk9xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i9 += 24;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
- const __m128i vxi9xGHIJKLMN = _mm_unpacklo_epi8(vi9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9xGHIJKLMN));
- const __m128i vxk9xGHIJKLMN = _mm_unpacklo_epi8(vk9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9xGHIJKLMN));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
+ const __m128i vxi9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi9xGHIJKLMN, vi9xGHIJKLMN), 8);
+ const __m128i vxk9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk9xGHIJKLMN, vk9xGHIJKLMN), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -466,12 +466,12 @@
const __m128i vk10xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 256 * sizeof(int8_t)));
i10 += 24;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
- const __m128i vxi10xGHIJKLMN = _mm_unpacklo_epi8(vi10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10xGHIJKLMN));
- const __m128i vxk10xGHIJKLMN = _mm_unpacklo_epi8(vk10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10xGHIJKLMN));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
+ const __m128i vxi10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi10xGHIJKLMN, vi10xGHIJKLMN), 8);
+ const __m128i vxk10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk10xGHIJKLMN, vk10xGHIJKLMN), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -495,12 +495,12 @@
const __m128i vk11xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i11 += 24;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
- const __m128i vxi11xGHIJKLMN = _mm_unpacklo_epi8(vi11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11xGHIJKLMN));
- const __m128i vxk11xGHIJKLMN = _mm_unpacklo_epi8(vk11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11xGHIJKLMN));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
+ const __m128i vxi11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi11xGHIJKLMN, vi11xGHIJKLMN), 8);
+ const __m128i vxk11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk11xGHIJKLMN, vk11xGHIJKLMN), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -524,12 +524,12 @@
const __m128i vk12xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 304 * sizeof(int8_t)));
i12 += 24;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
- const __m128i vxi12xGHIJKLMN = _mm_unpacklo_epi8(vi12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12xGHIJKLMN));
- const __m128i vxk12xGHIJKLMN = _mm_unpacklo_epi8(vk12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12xGHIJKLMN));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
+ const __m128i vxi12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi12xGHIJKLMN, vi12xGHIJKLMN), 8);
+ const __m128i vxk12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk12xGHIJKLMN, vk12xGHIJKLMN), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -553,12 +553,12 @@
const __m128i vk13xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i13 += 24;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
- const __m128i vxi13xGHIJKLMN = _mm_unpacklo_epi8(vi13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13xGHIJKLMN));
- const __m128i vxk13xGHIJKLMN = _mm_unpacklo_epi8(vk13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13xGHIJKLMN));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
+ const __m128i vxi13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi13xGHIJKLMN, vi13xGHIJKLMN), 8);
+ const __m128i vxk13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk13xGHIJKLMN, vk13xGHIJKLMN), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -582,12 +582,12 @@
const __m128i vk14xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 352 * sizeof(int8_t)));
i14 += 24;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
- const __m128i vxi14xGHIJKLMN = _mm_unpacklo_epi8(vi14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14xGHIJKLMN));
- const __m128i vxk14xGHIJKLMN = _mm_unpacklo_epi8(vk14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14xGHIJKLMN));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
+ const __m128i vxi14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi14xGHIJKLMN, vi14xGHIJKLMN), 8);
+ const __m128i vxk14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk14xGHIJKLMN, vk14xGHIJKLMN), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -611,12 +611,12 @@
const __m128i vk15xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i15 += 24;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
- const __m128i vxi15xGHIJKLMN = _mm_unpacklo_epi8(vi15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15xGHIJKLMN));
- const __m128i vxk15xGHIJKLMN = _mm_unpacklo_epi8(vk15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15xGHIJKLMN));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
+ const __m128i vxi15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi15xGHIJKLMN, vi15xGHIJKLMN), 8);
+ const __m128i vxk15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk15xGHIJKLMN, vk15xGHIJKLMN), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -640,12 +640,12 @@
const __m128i vk16xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 400 * sizeof(int8_t)));
i16 += 24;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
- const __m128i vxi16xGHIJKLMN = _mm_unpacklo_epi8(vi16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16xGHIJKLMN));
- const __m128i vxk16xGHIJKLMN = _mm_unpacklo_epi8(vk16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16xGHIJKLMN));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
+ const __m128i vxi16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi16xGHIJKLMN, vi16xGHIJKLMN), 8);
+ const __m128i vxk16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk16xGHIJKLMN, vk16xGHIJKLMN), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -669,12 +669,12 @@
const __m128i vk17xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 424 * sizeof(int8_t)));
i17 += 24;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
- const __m128i vxi17xGHIJKLMN = _mm_unpacklo_epi8(vi17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17xGHIJKLMN));
- const __m128i vxk17xGHIJKLMN = _mm_unpacklo_epi8(vk17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17xGHIJKLMN));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
+ const __m128i vxi17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi17xGHIJKLMN, vi17xGHIJKLMN), 8);
+ const __m128i vxk17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk17xGHIJKLMN, vk17xGHIJKLMN), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -698,12 +698,12 @@
const __m128i vk18xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 448 * sizeof(int8_t)));
i18 += 24;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
- const __m128i vxi18xGHIJKLMN = _mm_unpacklo_epi8(vi18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18xGHIJKLMN));
- const __m128i vxk18xGHIJKLMN = _mm_unpacklo_epi8(vk18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18xGHIJKLMN));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
+ const __m128i vxi18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi18xGHIJKLMN, vi18xGHIJKLMN), 8);
+ const __m128i vxk18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk18xGHIJKLMN, vk18xGHIJKLMN), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -727,12 +727,12 @@
const __m128i vk19xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 472 * sizeof(int8_t)));
i19 += 24;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
- const __m128i vxi19xGHIJKLMN = _mm_unpacklo_epi8(vi19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19xGHIJKLMN));
- const __m128i vxk19xGHIJKLMN = _mm_unpacklo_epi8(vk19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19xGHIJKLMN));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
+ const __m128i vxi19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi19xGHIJKLMN, vi19xGHIJKLMN), 8);
+ const __m128i vxk19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk19xGHIJKLMN, vk19xGHIJKLMN), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -756,12 +756,12 @@
const __m128i vk20xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 496 * sizeof(int8_t)));
i20 += 24;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
- const __m128i vxi20xGHIJKLMN = _mm_unpacklo_epi8(vi20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20xGHIJKLMN));
- const __m128i vxk20xGHIJKLMN = _mm_unpacklo_epi8(vk20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20xGHIJKLMN));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
+ const __m128i vxi20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi20xGHIJKLMN, vi20xGHIJKLMN), 8);
+ const __m128i vxk20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk20xGHIJKLMN, vk20xGHIJKLMN), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -785,12 +785,12 @@
const __m128i vk21xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 520 * sizeof(int8_t)));
i21 += 24;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
- const __m128i vxi21xGHIJKLMN = _mm_unpacklo_epi8(vi21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21xGHIJKLMN));
- const __m128i vxk21xGHIJKLMN = _mm_unpacklo_epi8(vk21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21xGHIJKLMN));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
+ const __m128i vxi21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi21xGHIJKLMN, vi21xGHIJKLMN), 8);
+ const __m128i vxk21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk21xGHIJKLMN, vk21xGHIJKLMN), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -814,12 +814,12 @@
const __m128i vk22xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 544 * sizeof(int8_t)));
i22 += 24;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
- const __m128i vxi22xGHIJKLMN = _mm_unpacklo_epi8(vi22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22xGHIJKLMN));
- const __m128i vxk22xGHIJKLMN = _mm_unpacklo_epi8(vk22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22xGHIJKLMN));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
+ const __m128i vxi22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi22xGHIJKLMN, vi22xGHIJKLMN), 8);
+ const __m128i vxk22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk22xGHIJKLMN, vk22xGHIJKLMN), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -843,12 +843,12 @@
const __m128i vk23xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 568 * sizeof(int8_t)));
i23 += 24;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
- const __m128i vxi23xGHIJKLMN = _mm_unpacklo_epi8(vi23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23xGHIJKLMN));
- const __m128i vxk23xGHIJKLMN = _mm_unpacklo_epi8(vk23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23xGHIJKLMN));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
+ const __m128i vxi23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi23xGHIJKLMN, vi23xGHIJKLMN), 8);
+ const __m128i vxk23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk23xGHIJKLMN, vk23xGHIJKLMN), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -872,12 +872,12 @@
const __m128i vk24xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 592 * sizeof(int8_t)));
i24 += 24;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
- const __m128i vxi24xGHIJKLMN = _mm_unpacklo_epi8(vi24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24xGHIJKLMN));
- const __m128i vxk24xGHIJKLMN = _mm_unpacklo_epi8(vk24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24xGHIJKLMN));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
+ const __m128i vxi24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi24xGHIJKLMN, vi24xGHIJKLMN), 8);
+ const __m128i vxk24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk24xGHIJKLMN, vk24xGHIJKLMN), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1043,18 +1043,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -1066,8 +1066,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -1079,8 +1079,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -1092,8 +1092,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -1105,8 +1105,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -1118,8 +1118,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -1131,8 +1131,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -1144,8 +1144,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -1157,8 +1157,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -1170,8 +1170,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 216));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -1183,8 +1183,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -1196,8 +1196,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 264));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -1209,8 +1209,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -1222,8 +1222,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 312));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -1235,8 +1235,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -1248,8 +1248,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 360));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -1261,8 +1261,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -1274,8 +1274,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 408));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -1287,8 +1287,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 432));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -1300,8 +1300,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 456));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -1313,8 +1313,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 480));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1326,8 +1326,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 504));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1339,8 +1339,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 528));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1352,8 +1352,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 552));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1365,8 +1365,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 576));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1429,7 +1429,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c
index 3b64afb..d92f329 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -988,10 +988,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -1354,7 +1354,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c
index 94666ce..acbdb13 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-sse41-mul32.c
@@ -162,11 +162,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -786,7 +786,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -965,7 +965,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c
index 958c2ee..6279be3 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-ssse3-mul16.c
@@ -161,11 +161,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -176,12 +176,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -205,12 +205,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -234,12 +234,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -263,12 +263,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -292,12 +292,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -321,12 +321,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -350,12 +350,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -379,12 +379,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -408,12 +408,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -437,12 +437,12 @@
const __m128i vk9xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 232 * sizeof(int8_t)));
i9 += 24;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
- const __m128i vxi9x89ABCDEF = _mm_unpacklo_epi8(vi9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x89ABCDEF));
- const __m128i vxk9x89ABCDEF = _mm_unpacklo_epi8(vk9x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x89ABCDEF));
- const __m128i vxi9xGHIJKLMN = _mm_unpacklo_epi8(vi9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9xGHIJKLMN));
- const __m128i vxk9xGHIJKLMN = _mm_unpacklo_epi8(vk9xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9xGHIJKLMN));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
+ const __m128i vxi9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x89ABCDEF, vi9x89ABCDEF), 8);
+ const __m128i vxk9x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x89ABCDEF, vk9x89ABCDEF), 8);
+ const __m128i vxi9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi9xGHIJKLMN, vi9xGHIJKLMN), 8);
+ const __m128i vxk9xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk9xGHIJKLMN, vk9xGHIJKLMN), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -466,12 +466,12 @@
const __m128i vk10xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 256 * sizeof(int8_t)));
i10 += 24;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
- const __m128i vxi10x89ABCDEF = _mm_unpacklo_epi8(vi10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x89ABCDEF));
- const __m128i vxk10x89ABCDEF = _mm_unpacklo_epi8(vk10x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x89ABCDEF));
- const __m128i vxi10xGHIJKLMN = _mm_unpacklo_epi8(vi10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10xGHIJKLMN));
- const __m128i vxk10xGHIJKLMN = _mm_unpacklo_epi8(vk10xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10xGHIJKLMN));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
+ const __m128i vxi10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x89ABCDEF, vi10x89ABCDEF), 8);
+ const __m128i vxk10x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x89ABCDEF, vk10x89ABCDEF), 8);
+ const __m128i vxi10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi10xGHIJKLMN, vi10xGHIJKLMN), 8);
+ const __m128i vxk10xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk10xGHIJKLMN, vk10xGHIJKLMN), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -495,12 +495,12 @@
const __m128i vk11xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 280 * sizeof(int8_t)));
i11 += 24;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
- const __m128i vxi11x89ABCDEF = _mm_unpacklo_epi8(vi11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x89ABCDEF));
- const __m128i vxk11x89ABCDEF = _mm_unpacklo_epi8(vk11x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x89ABCDEF));
- const __m128i vxi11xGHIJKLMN = _mm_unpacklo_epi8(vi11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11xGHIJKLMN));
- const __m128i vxk11xGHIJKLMN = _mm_unpacklo_epi8(vk11xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11xGHIJKLMN));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
+ const __m128i vxi11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x89ABCDEF, vi11x89ABCDEF), 8);
+ const __m128i vxk11x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x89ABCDEF, vk11x89ABCDEF), 8);
+ const __m128i vxi11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi11xGHIJKLMN, vi11xGHIJKLMN), 8);
+ const __m128i vxk11xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk11xGHIJKLMN, vk11xGHIJKLMN), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -524,12 +524,12 @@
const __m128i vk12xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 304 * sizeof(int8_t)));
i12 += 24;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
- const __m128i vxi12x89ABCDEF = _mm_unpacklo_epi8(vi12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x89ABCDEF));
- const __m128i vxk12x89ABCDEF = _mm_unpacklo_epi8(vk12x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x89ABCDEF));
- const __m128i vxi12xGHIJKLMN = _mm_unpacklo_epi8(vi12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12xGHIJKLMN));
- const __m128i vxk12xGHIJKLMN = _mm_unpacklo_epi8(vk12xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12xGHIJKLMN));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
+ const __m128i vxi12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x89ABCDEF, vi12x89ABCDEF), 8);
+ const __m128i vxk12x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x89ABCDEF, vk12x89ABCDEF), 8);
+ const __m128i vxi12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi12xGHIJKLMN, vi12xGHIJKLMN), 8);
+ const __m128i vxk12xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk12xGHIJKLMN, vk12xGHIJKLMN), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -553,12 +553,12 @@
const __m128i vk13xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 328 * sizeof(int8_t)));
i13 += 24;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
- const __m128i vxi13x89ABCDEF = _mm_unpacklo_epi8(vi13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x89ABCDEF));
- const __m128i vxk13x89ABCDEF = _mm_unpacklo_epi8(vk13x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x89ABCDEF));
- const __m128i vxi13xGHIJKLMN = _mm_unpacklo_epi8(vi13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13xGHIJKLMN));
- const __m128i vxk13xGHIJKLMN = _mm_unpacklo_epi8(vk13xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13xGHIJKLMN));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
+ const __m128i vxi13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x89ABCDEF, vi13x89ABCDEF), 8);
+ const __m128i vxk13x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x89ABCDEF, vk13x89ABCDEF), 8);
+ const __m128i vxi13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi13xGHIJKLMN, vi13xGHIJKLMN), 8);
+ const __m128i vxk13xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk13xGHIJKLMN, vk13xGHIJKLMN), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -582,12 +582,12 @@
const __m128i vk14xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 352 * sizeof(int8_t)));
i14 += 24;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
- const __m128i vxi14x89ABCDEF = _mm_unpacklo_epi8(vi14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x89ABCDEF));
- const __m128i vxk14x89ABCDEF = _mm_unpacklo_epi8(vk14x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x89ABCDEF));
- const __m128i vxi14xGHIJKLMN = _mm_unpacklo_epi8(vi14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14xGHIJKLMN));
- const __m128i vxk14xGHIJKLMN = _mm_unpacklo_epi8(vk14xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14xGHIJKLMN));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
+ const __m128i vxi14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x89ABCDEF, vi14x89ABCDEF), 8);
+ const __m128i vxk14x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x89ABCDEF, vk14x89ABCDEF), 8);
+ const __m128i vxi14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi14xGHIJKLMN, vi14xGHIJKLMN), 8);
+ const __m128i vxk14xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk14xGHIJKLMN, vk14xGHIJKLMN), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -611,12 +611,12 @@
const __m128i vk15xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 376 * sizeof(int8_t)));
i15 += 24;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
- const __m128i vxi15x89ABCDEF = _mm_unpacklo_epi8(vi15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x89ABCDEF));
- const __m128i vxk15x89ABCDEF = _mm_unpacklo_epi8(vk15x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x89ABCDEF));
- const __m128i vxi15xGHIJKLMN = _mm_unpacklo_epi8(vi15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15xGHIJKLMN));
- const __m128i vxk15xGHIJKLMN = _mm_unpacklo_epi8(vk15xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15xGHIJKLMN));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
+ const __m128i vxi15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x89ABCDEF, vi15x89ABCDEF), 8);
+ const __m128i vxk15x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x89ABCDEF, vk15x89ABCDEF), 8);
+ const __m128i vxi15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi15xGHIJKLMN, vi15xGHIJKLMN), 8);
+ const __m128i vxk15xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk15xGHIJKLMN, vk15xGHIJKLMN), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -640,12 +640,12 @@
const __m128i vk16xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 400 * sizeof(int8_t)));
i16 += 24;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
- const __m128i vxi16x89ABCDEF = _mm_unpacklo_epi8(vi16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x89ABCDEF));
- const __m128i vxk16x89ABCDEF = _mm_unpacklo_epi8(vk16x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x89ABCDEF));
- const __m128i vxi16xGHIJKLMN = _mm_unpacklo_epi8(vi16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16xGHIJKLMN));
- const __m128i vxk16xGHIJKLMN = _mm_unpacklo_epi8(vk16xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16xGHIJKLMN));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
+ const __m128i vxi16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x89ABCDEF, vi16x89ABCDEF), 8);
+ const __m128i vxk16x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x89ABCDEF, vk16x89ABCDEF), 8);
+ const __m128i vxi16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi16xGHIJKLMN, vi16xGHIJKLMN), 8);
+ const __m128i vxk16xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk16xGHIJKLMN, vk16xGHIJKLMN), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -669,12 +669,12 @@
const __m128i vk17xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 424 * sizeof(int8_t)));
i17 += 24;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
- const __m128i vxi17x89ABCDEF = _mm_unpacklo_epi8(vi17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x89ABCDEF));
- const __m128i vxk17x89ABCDEF = _mm_unpacklo_epi8(vk17x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x89ABCDEF));
- const __m128i vxi17xGHIJKLMN = _mm_unpacklo_epi8(vi17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17xGHIJKLMN));
- const __m128i vxk17xGHIJKLMN = _mm_unpacklo_epi8(vk17xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17xGHIJKLMN));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
+ const __m128i vxi17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x89ABCDEF, vi17x89ABCDEF), 8);
+ const __m128i vxk17x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x89ABCDEF, vk17x89ABCDEF), 8);
+ const __m128i vxi17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi17xGHIJKLMN, vi17xGHIJKLMN), 8);
+ const __m128i vxk17xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk17xGHIJKLMN, vk17xGHIJKLMN), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -698,12 +698,12 @@
const __m128i vk18xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 448 * sizeof(int8_t)));
i18 += 24;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
- const __m128i vxi18x89ABCDEF = _mm_unpacklo_epi8(vi18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x89ABCDEF));
- const __m128i vxk18x89ABCDEF = _mm_unpacklo_epi8(vk18x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x89ABCDEF));
- const __m128i vxi18xGHIJKLMN = _mm_unpacklo_epi8(vi18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18xGHIJKLMN));
- const __m128i vxk18xGHIJKLMN = _mm_unpacklo_epi8(vk18xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18xGHIJKLMN));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
+ const __m128i vxi18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x89ABCDEF, vi18x89ABCDEF), 8);
+ const __m128i vxk18x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x89ABCDEF, vk18x89ABCDEF), 8);
+ const __m128i vxi18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi18xGHIJKLMN, vi18xGHIJKLMN), 8);
+ const __m128i vxk18xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk18xGHIJKLMN, vk18xGHIJKLMN), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -727,12 +727,12 @@
const __m128i vk19xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 472 * sizeof(int8_t)));
i19 += 24;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
- const __m128i vxi19x89ABCDEF = _mm_unpacklo_epi8(vi19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x89ABCDEF));
- const __m128i vxk19x89ABCDEF = _mm_unpacklo_epi8(vk19x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x89ABCDEF));
- const __m128i vxi19xGHIJKLMN = _mm_unpacklo_epi8(vi19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19xGHIJKLMN));
- const __m128i vxk19xGHIJKLMN = _mm_unpacklo_epi8(vk19xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19xGHIJKLMN));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
+ const __m128i vxi19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x89ABCDEF, vi19x89ABCDEF), 8);
+ const __m128i vxk19x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x89ABCDEF, vk19x89ABCDEF), 8);
+ const __m128i vxi19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi19xGHIJKLMN, vi19xGHIJKLMN), 8);
+ const __m128i vxk19xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk19xGHIJKLMN, vk19xGHIJKLMN), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -756,12 +756,12 @@
const __m128i vk20xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 496 * sizeof(int8_t)));
i20 += 24;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
- const __m128i vxi20x89ABCDEF = _mm_unpacklo_epi8(vi20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x89ABCDEF));
- const __m128i vxk20x89ABCDEF = _mm_unpacklo_epi8(vk20x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x89ABCDEF));
- const __m128i vxi20xGHIJKLMN = _mm_unpacklo_epi8(vi20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20xGHIJKLMN));
- const __m128i vxk20xGHIJKLMN = _mm_unpacklo_epi8(vk20xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20xGHIJKLMN));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
+ const __m128i vxi20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x89ABCDEF, vi20x89ABCDEF), 8);
+ const __m128i vxk20x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x89ABCDEF, vk20x89ABCDEF), 8);
+ const __m128i vxi20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi20xGHIJKLMN, vi20xGHIJKLMN), 8);
+ const __m128i vxk20xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk20xGHIJKLMN, vk20xGHIJKLMN), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -785,12 +785,12 @@
const __m128i vk21xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 520 * sizeof(int8_t)));
i21 += 24;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
- const __m128i vxi21x89ABCDEF = _mm_unpacklo_epi8(vi21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x89ABCDEF));
- const __m128i vxk21x89ABCDEF = _mm_unpacklo_epi8(vk21x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x89ABCDEF));
- const __m128i vxi21xGHIJKLMN = _mm_unpacklo_epi8(vi21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21xGHIJKLMN));
- const __m128i vxk21xGHIJKLMN = _mm_unpacklo_epi8(vk21xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21xGHIJKLMN));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
+ const __m128i vxi21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x89ABCDEF, vi21x89ABCDEF), 8);
+ const __m128i vxk21x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x89ABCDEF, vk21x89ABCDEF), 8);
+ const __m128i vxi21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi21xGHIJKLMN, vi21xGHIJKLMN), 8);
+ const __m128i vxk21xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk21xGHIJKLMN, vk21xGHIJKLMN), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -814,12 +814,12 @@
const __m128i vk22xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 544 * sizeof(int8_t)));
i22 += 24;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
- const __m128i vxi22x89ABCDEF = _mm_unpacklo_epi8(vi22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x89ABCDEF));
- const __m128i vxk22x89ABCDEF = _mm_unpacklo_epi8(vk22x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x89ABCDEF));
- const __m128i vxi22xGHIJKLMN = _mm_unpacklo_epi8(vi22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22xGHIJKLMN));
- const __m128i vxk22xGHIJKLMN = _mm_unpacklo_epi8(vk22xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22xGHIJKLMN));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
+ const __m128i vxi22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x89ABCDEF, vi22x89ABCDEF), 8);
+ const __m128i vxk22x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x89ABCDEF, vk22x89ABCDEF), 8);
+ const __m128i vxi22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi22xGHIJKLMN, vi22xGHIJKLMN), 8);
+ const __m128i vxk22xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk22xGHIJKLMN, vk22xGHIJKLMN), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -843,12 +843,12 @@
const __m128i vk23xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 568 * sizeof(int8_t)));
i23 += 24;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
- const __m128i vxi23x89ABCDEF = _mm_unpacklo_epi8(vi23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x89ABCDEF));
- const __m128i vxk23x89ABCDEF = _mm_unpacklo_epi8(vk23x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x89ABCDEF));
- const __m128i vxi23xGHIJKLMN = _mm_unpacklo_epi8(vi23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23xGHIJKLMN));
- const __m128i vxk23xGHIJKLMN = _mm_unpacklo_epi8(vk23xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23xGHIJKLMN));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
+ const __m128i vxi23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x89ABCDEF, vi23x89ABCDEF), 8);
+ const __m128i vxk23x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x89ABCDEF, vk23x89ABCDEF), 8);
+ const __m128i vxi23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi23xGHIJKLMN, vi23xGHIJKLMN), 8);
+ const __m128i vxk23xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk23xGHIJKLMN, vk23xGHIJKLMN), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -872,12 +872,12 @@
const __m128i vk24xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 592 * sizeof(int8_t)));
i24 += 24;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
- const __m128i vxi24x89ABCDEF = _mm_unpacklo_epi8(vi24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x89ABCDEF));
- const __m128i vxk24x89ABCDEF = _mm_unpacklo_epi8(vk24x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x89ABCDEF));
- const __m128i vxi24xGHIJKLMN = _mm_unpacklo_epi8(vi24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24xGHIJKLMN));
- const __m128i vxk24xGHIJKLMN = _mm_unpacklo_epi8(vk24xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24xGHIJKLMN));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
+ const __m128i vxi24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x89ABCDEF, vi24x89ABCDEF), 8);
+ const __m128i vxk24x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x89ABCDEF, vk24x89ABCDEF), 8);
+ const __m128i vxi24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi24xGHIJKLMN, vi24xGHIJKLMN), 8);
+ const __m128i vxk24xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk24xGHIJKLMN, vk24xGHIJKLMN), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1043,18 +1043,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -1066,8 +1066,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -1079,8 +1079,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -1092,8 +1092,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -1105,8 +1105,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -1118,8 +1118,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -1131,8 +1131,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -1144,8 +1144,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -1157,8 +1157,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -1170,8 +1170,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) (k + 216));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -1183,8 +1183,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) (k + 240));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -1196,8 +1196,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) (k + 264));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -1209,8 +1209,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) (k + 288));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -1222,8 +1222,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) (k + 312));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -1235,8 +1235,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) (k + 336));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -1248,8 +1248,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) (k + 360));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -1261,8 +1261,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) (k + 384));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -1274,8 +1274,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) (k + 408));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -1287,8 +1287,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) (k + 432));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -1300,8 +1300,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) (k + 456));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -1313,8 +1313,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) (k + 480));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -1326,8 +1326,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) (k + 504));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -1339,8 +1339,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) (k + 528));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -1352,8 +1352,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) (k + 552));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -1365,8 +1365,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) (k + 576));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -1429,7 +1429,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c
index e16c6bf..16c1aec 100644
--- a/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x25-minmax-gemmlowp-xop-mul32.c
@@ -167,11 +167,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -791,7 +791,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -970,7 +970,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c
index 8adb1e5..482abb3 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -395,10 +395,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -530,7 +530,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c
index c4b4708..55bdc07 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-avx-mul32.c
@@ -82,11 +82,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -321,7 +321,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -386,7 +386,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c
index 7f92408..edd1908 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse2-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -96,12 +96,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -125,12 +125,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -154,12 +154,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -183,12 +183,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -212,12 +212,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -241,12 +241,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -270,12 +270,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -299,12 +299,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -328,12 +328,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -397,18 +397,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -420,8 +420,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -433,8 +433,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -446,8 +446,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -459,8 +459,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -472,8 +472,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -485,8 +485,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -498,8 +498,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -511,8 +511,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -532,7 +532,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c
index df7fe14..913fc8f 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -395,10 +395,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -530,7 +530,7 @@
vacc0123 = _mm_cvtps_epi32(vscaled0123);
vacc4567 = _mm_cvtps_epi32(vscaled4567);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c
index 4c0f1fe..685466d 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-sse41-mul32.c
@@ -82,11 +82,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -321,7 +321,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -386,7 +386,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c
index b66d970..7da9e42 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-fp32-xop-mul32.c
@@ -87,11 +87,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -326,7 +326,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -391,7 +391,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c
index fbd1e5d..a180fe4 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -444,10 +444,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -602,7 +602,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c
index f7991c9..709d965 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-avx-mul32.c
@@ -82,11 +82,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -370,7 +370,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -453,7 +453,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c
index 1e4e69f..e44f905 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse2-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -96,12 +96,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -125,12 +125,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -154,12 +154,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -183,12 +183,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -212,12 +212,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -241,12 +241,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -270,12 +270,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -299,12 +299,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -328,12 +328,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -499,18 +499,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -522,8 +522,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -535,8 +535,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -548,8 +548,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -561,8 +561,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -574,8 +574,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -587,8 +587,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -600,8 +600,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -613,8 +613,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -677,7 +677,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c
index b049fad..931efed 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -444,10 +444,10 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -602,7 +602,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c
index 08d42b1..68dbff7 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-sse41-mul32.c
@@ -82,11 +82,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -370,7 +370,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -453,7 +453,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c
index 9e3dbc3..ab80f73 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-ssse3-mul16.c
@@ -81,11 +81,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -96,12 +96,12 @@
const __m128i vk0xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i0 += 24;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
- const __m128i vxi0x89ABCDEF = _mm_unpacklo_epi8(vi0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x89ABCDEF));
- const __m128i vxk0x89ABCDEF = _mm_unpacklo_epi8(vk0x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x89ABCDEF));
- const __m128i vxi0xGHIJKLMN = _mm_unpacklo_epi8(vi0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0xGHIJKLMN));
- const __m128i vxk0xGHIJKLMN = _mm_unpacklo_epi8(vk0xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0xGHIJKLMN));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
+ const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
+ const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
+ const __m128i vxi0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi0xGHIJKLMN, vi0xGHIJKLMN), 8);
+ const __m128i vxk0xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk0xGHIJKLMN, vk0xGHIJKLMN), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -125,12 +125,12 @@
const __m128i vk1xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i1 += 24;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
- const __m128i vxi1x89ABCDEF = _mm_unpacklo_epi8(vi1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x89ABCDEF));
- const __m128i vxk1x89ABCDEF = _mm_unpacklo_epi8(vk1x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x89ABCDEF));
- const __m128i vxi1xGHIJKLMN = _mm_unpacklo_epi8(vi1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1xGHIJKLMN));
- const __m128i vxk1xGHIJKLMN = _mm_unpacklo_epi8(vk1xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1xGHIJKLMN));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
+ const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
+ const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
+ const __m128i vxi1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi1xGHIJKLMN, vi1xGHIJKLMN), 8);
+ const __m128i vxk1xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk1xGHIJKLMN, vk1xGHIJKLMN), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -154,12 +154,12 @@
const __m128i vk2xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i2 += 24;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
- const __m128i vxi2x89ABCDEF = _mm_unpacklo_epi8(vi2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x89ABCDEF));
- const __m128i vxk2x89ABCDEF = _mm_unpacklo_epi8(vk2x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x89ABCDEF));
- const __m128i vxi2xGHIJKLMN = _mm_unpacklo_epi8(vi2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2xGHIJKLMN));
- const __m128i vxk2xGHIJKLMN = _mm_unpacklo_epi8(vk2xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2xGHIJKLMN));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
+ const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
+ const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
+ const __m128i vxi2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi2xGHIJKLMN, vi2xGHIJKLMN), 8);
+ const __m128i vxk2xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk2xGHIJKLMN, vk2xGHIJKLMN), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -183,12 +183,12 @@
const __m128i vk3xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i3 += 24;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
- const __m128i vxi3x89ABCDEF = _mm_unpacklo_epi8(vi3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x89ABCDEF));
- const __m128i vxk3x89ABCDEF = _mm_unpacklo_epi8(vk3x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x89ABCDEF));
- const __m128i vxi3xGHIJKLMN = _mm_unpacklo_epi8(vi3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3xGHIJKLMN));
- const __m128i vxk3xGHIJKLMN = _mm_unpacklo_epi8(vk3xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3xGHIJKLMN));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
+ const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
+ const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
+ const __m128i vxi3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi3xGHIJKLMN, vi3xGHIJKLMN), 8);
+ const __m128i vxk3xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk3xGHIJKLMN, vk3xGHIJKLMN), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -212,12 +212,12 @@
const __m128i vk4xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i4 += 24;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
- const __m128i vxi4x89ABCDEF = _mm_unpacklo_epi8(vi4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x89ABCDEF));
- const __m128i vxk4x89ABCDEF = _mm_unpacklo_epi8(vk4x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x89ABCDEF));
- const __m128i vxi4xGHIJKLMN = _mm_unpacklo_epi8(vi4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4xGHIJKLMN));
- const __m128i vxk4xGHIJKLMN = _mm_unpacklo_epi8(vk4xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4xGHIJKLMN));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
+ const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
+ const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
+ const __m128i vxi4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi4xGHIJKLMN, vi4xGHIJKLMN), 8);
+ const __m128i vxk4xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk4xGHIJKLMN, vk4xGHIJKLMN), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -241,12 +241,12 @@
const __m128i vk5xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i5 += 24;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
- const __m128i vxi5x89ABCDEF = _mm_unpacklo_epi8(vi5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x89ABCDEF));
- const __m128i vxk5x89ABCDEF = _mm_unpacklo_epi8(vk5x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x89ABCDEF));
- const __m128i vxi5xGHIJKLMN = _mm_unpacklo_epi8(vi5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5xGHIJKLMN));
- const __m128i vxk5xGHIJKLMN = _mm_unpacklo_epi8(vk5xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5xGHIJKLMN));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
+ const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
+ const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
+ const __m128i vxi5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi5xGHIJKLMN, vi5xGHIJKLMN), 8);
+ const __m128i vxk5xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk5xGHIJKLMN, vk5xGHIJKLMN), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -270,12 +270,12 @@
const __m128i vk6xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i6 += 24;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
- const __m128i vxi6x89ABCDEF = _mm_unpacklo_epi8(vi6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x89ABCDEF));
- const __m128i vxk6x89ABCDEF = _mm_unpacklo_epi8(vk6x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x89ABCDEF));
- const __m128i vxi6xGHIJKLMN = _mm_unpacklo_epi8(vi6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6xGHIJKLMN));
- const __m128i vxk6xGHIJKLMN = _mm_unpacklo_epi8(vk6xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6xGHIJKLMN));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
+ const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
+ const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
+ const __m128i vxi6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi6xGHIJKLMN, vi6xGHIJKLMN), 8);
+ const __m128i vxk6xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk6xGHIJKLMN, vk6xGHIJKLMN), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -299,12 +299,12 @@
const __m128i vk7xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i7 += 24;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
- const __m128i vxi7x89ABCDEF = _mm_unpacklo_epi8(vi7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x89ABCDEF));
- const __m128i vxk7x89ABCDEF = _mm_unpacklo_epi8(vk7x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x89ABCDEF));
- const __m128i vxi7xGHIJKLMN = _mm_unpacklo_epi8(vi7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7xGHIJKLMN));
- const __m128i vxk7xGHIJKLMN = _mm_unpacklo_epi8(vk7xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7xGHIJKLMN));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
+ const __m128i vxi7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x89ABCDEF, vi7x89ABCDEF), 8);
+ const __m128i vxk7x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x89ABCDEF, vk7x89ABCDEF), 8);
+ const __m128i vxi7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi7xGHIJKLMN, vi7xGHIJKLMN), 8);
+ const __m128i vxk7xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk7xGHIJKLMN, vk7xGHIJKLMN), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -328,12 +328,12 @@
const __m128i vk8xGHIJKLMN = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int32_t) + 208 * sizeof(int8_t)));
i8 += 24;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
- const __m128i vxi8x89ABCDEF = _mm_unpacklo_epi8(vi8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x89ABCDEF));
- const __m128i vxk8x89ABCDEF = _mm_unpacklo_epi8(vk8x89ABCDEF, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x89ABCDEF));
- const __m128i vxi8xGHIJKLMN = _mm_unpacklo_epi8(vi8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8xGHIJKLMN));
- const __m128i vxk8xGHIJKLMN = _mm_unpacklo_epi8(vk8xGHIJKLMN, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8xGHIJKLMN));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
+ const __m128i vxi8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x89ABCDEF, vi8x89ABCDEF), 8);
+ const __m128i vxk8x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x89ABCDEF, vk8x89ABCDEF), 8);
+ const __m128i vxi8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vi8xGHIJKLMN, vi8xGHIJKLMN), 8);
+ const __m128i vxk8xGHIJKLMN = _mm_srai_epi16(_mm_unpacklo_epi8(vk8xGHIJKLMN, vk8xGHIJKLMN), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -499,18 +499,18 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) k);
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -522,8 +522,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) (k + 24));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -535,8 +535,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) (k + 48));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -548,8 +548,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) (k + 72));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -561,8 +561,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) (k + 96));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -574,8 +574,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) (k + 120));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -587,8 +587,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) (k + 144));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -600,8 +600,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) (k + 168));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -613,8 +613,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) (k + 192));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -677,7 +677,7 @@
vacc4567 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod4567, vshift), _mm_cmpgt_epi32(vrem4567, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse2.output_zero_point);
__m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c
index c879c5a..eea6e4d 100644
--- a/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up24x9-minmax-gemmlowp-xop-mul32.c
@@ -87,11 +87,11 @@
const void* w = weights;
for (; c >= 24; c -= 24) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
- __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
- __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 12 * sizeof(int32_t)));
- __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
- __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 20 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
+ __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
+ __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
+ __m128i vaccGHIJ = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 16));
+ __m128i vaccKLMN = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 20));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -375,7 +375,7 @@
output += 24;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 24 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 24);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -458,7 +458,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c
index a4a4476..d7af3dc 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul16.c
@@ -161,7 +161,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -519,7 +519,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c
index f96d6e3..793b4d7 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-avx-mul32.c
@@ -162,7 +162,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -415,7 +415,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -576,7 +576,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c
index 990ba4b..d800cb3 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse2-mul16.c
@@ -161,15 +161,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -181,8 +181,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -194,8 +194,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -207,8 +207,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -220,8 +220,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -233,8 +233,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -246,8 +246,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -259,8 +259,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -272,8 +272,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -285,8 +285,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -298,8 +298,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -311,8 +311,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -324,8 +324,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -337,8 +337,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -350,8 +350,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -363,8 +363,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -376,8 +376,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -389,8 +389,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -402,8 +402,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -415,8 +415,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -428,8 +428,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -441,8 +441,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -454,8 +454,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -467,8 +467,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -480,8 +480,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -519,14 +519,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -537,8 +537,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -549,8 +549,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -561,8 +561,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -573,8 +573,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -585,8 +585,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -597,8 +597,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -609,8 +609,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -621,8 +621,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -633,8 +633,8 @@
const __m128i vi9x01234567 = _mm_loadl_epi64((const __m128i*) i9);
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -645,8 +645,8 @@
const __m128i vi10x01234567 = _mm_loadl_epi64((const __m128i*) i10);
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -657,8 +657,8 @@
const __m128i vi11x01234567 = _mm_loadl_epi64((const __m128i*) i11);
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -669,8 +669,8 @@
const __m128i vi12x01234567 = _mm_loadl_epi64((const __m128i*) i12);
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -681,8 +681,8 @@
const __m128i vi13x01234567 = _mm_loadl_epi64((const __m128i*) i13);
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -693,8 +693,8 @@
const __m128i vi14x01234567 = _mm_loadl_epi64((const __m128i*) i14);
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -705,8 +705,8 @@
const __m128i vi15x01234567 = _mm_loadl_epi64((const __m128i*) i15);
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -717,8 +717,8 @@
const __m128i vi16x01234567 = _mm_loadl_epi64((const __m128i*) i16);
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -729,8 +729,8 @@
const __m128i vi17x01234567 = _mm_loadl_epi64((const __m128i*) i17);
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -741,8 +741,8 @@
const __m128i vi18x01234567 = _mm_loadl_epi64((const __m128i*) i18);
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -753,8 +753,8 @@
const __m128i vi19x01234567 = _mm_loadl_epi64((const __m128i*) i19);
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -765,8 +765,8 @@
const __m128i vi20x01234567 = _mm_loadl_epi64((const __m128i*) i20);
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -777,8 +777,8 @@
const __m128i vi21x01234567 = _mm_loadl_epi64((const __m128i*) i21);
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -789,8 +789,8 @@
const __m128i vi22x01234567 = _mm_loadl_epi64((const __m128i*) i22);
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -801,8 +801,8 @@
const __m128i vi23x01234567 = _mm_loadl_epi64((const __m128i*) i23);
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -813,8 +813,8 @@
const __m128i vi24x01234567 = _mm_loadl_epi64((const __m128i*) i24);
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c
index 06545ff..4f47281 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul16.c
@@ -161,7 +161,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -519,7 +519,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c
index cf0706b..0e6e79f 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-sse41-mul32.c
@@ -162,7 +162,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -415,7 +415,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -576,7 +576,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c
index 4f4d8c3..a94e03d 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-fp32-xop-mul32.c
@@ -167,7 +167,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -420,7 +420,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -581,7 +581,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c
index 3eaab6b..5713140 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul16.c
@@ -161,7 +161,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -540,7 +540,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c
index 560fcee..e831243 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-avx-mul32.c
@@ -162,7 +162,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -436,7 +436,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -615,7 +615,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c
index 3f3fbee..8c0ce46 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse2-mul16.c
@@ -161,15 +161,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -181,8 +181,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -194,8 +194,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -207,8 +207,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -220,8 +220,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -233,8 +233,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -246,8 +246,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -259,8 +259,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -272,8 +272,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -285,8 +285,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -298,8 +298,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -311,8 +311,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -324,8 +324,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -337,8 +337,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -350,8 +350,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -363,8 +363,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -376,8 +376,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -389,8 +389,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -402,8 +402,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -415,8 +415,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -428,8 +428,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -441,8 +441,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -454,8 +454,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -467,8 +467,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -480,8 +480,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -561,14 +561,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -579,8 +579,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -591,8 +591,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -603,8 +603,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -615,8 +615,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -627,8 +627,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -639,8 +639,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -651,8 +651,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -663,8 +663,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -675,8 +675,8 @@
const __m128i vi9x01234567 = _mm_loadl_epi64((const __m128i*) i9);
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -687,8 +687,8 @@
const __m128i vi10x01234567 = _mm_loadl_epi64((const __m128i*) i10);
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -699,8 +699,8 @@
const __m128i vi11x01234567 = _mm_loadl_epi64((const __m128i*) i11);
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -711,8 +711,8 @@
const __m128i vi12x01234567 = _mm_loadl_epi64((const __m128i*) i12);
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -723,8 +723,8 @@
const __m128i vi13x01234567 = _mm_loadl_epi64((const __m128i*) i13);
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -735,8 +735,8 @@
const __m128i vi14x01234567 = _mm_loadl_epi64((const __m128i*) i14);
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -747,8 +747,8 @@
const __m128i vi15x01234567 = _mm_loadl_epi64((const __m128i*) i15);
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -759,8 +759,8 @@
const __m128i vi16x01234567 = _mm_loadl_epi64((const __m128i*) i16);
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -771,8 +771,8 @@
const __m128i vi17x01234567 = _mm_loadl_epi64((const __m128i*) i17);
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -783,8 +783,8 @@
const __m128i vi18x01234567 = _mm_loadl_epi64((const __m128i*) i18);
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -795,8 +795,8 @@
const __m128i vi19x01234567 = _mm_loadl_epi64((const __m128i*) i19);
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -807,8 +807,8 @@
const __m128i vi20x01234567 = _mm_loadl_epi64((const __m128i*) i20);
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -819,8 +819,8 @@
const __m128i vi21x01234567 = _mm_loadl_epi64((const __m128i*) i21);
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -831,8 +831,8 @@
const __m128i vi22x01234567 = _mm_loadl_epi64((const __m128i*) i22);
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -843,8 +843,8 @@
const __m128i vi23x01234567 = _mm_loadl_epi64((const __m128i*) i23);
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -855,8 +855,8 @@
const __m128i vi24x01234567 = _mm_loadl_epi64((const __m128i*) i24);
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c
index 455b43b..e8c2a2b 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul16.c
@@ -161,7 +161,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -540,7 +540,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c
index 2a5a926..fa2fe0c 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-sse41-mul32.c
@@ -162,7 +162,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -436,7 +436,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -615,7 +615,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c
index 9989362..ba6f057 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-ssse3-mul16.c
@@ -161,15 +161,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -181,8 +181,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -194,8 +194,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -207,8 +207,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -220,8 +220,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -233,8 +233,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -246,8 +246,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -259,8 +259,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -272,8 +272,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -285,8 +285,8 @@
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
i9 += 8;
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -298,8 +298,8 @@
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
i10 += 8;
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -311,8 +311,8 @@
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
i11 += 8;
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -324,8 +324,8 @@
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
i12 += 8;
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -337,8 +337,8 @@
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
i13 += 8;
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -350,8 +350,8 @@
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
i14 += 8;
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -363,8 +363,8 @@
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
i15 += 8;
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -376,8 +376,8 @@
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
i16 += 8;
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -389,8 +389,8 @@
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
i17 += 8;
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -402,8 +402,8 @@
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
i18 += 8;
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -415,8 +415,8 @@
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
i19 += 8;
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -428,8 +428,8 @@
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
i20 += 8;
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -441,8 +441,8 @@
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
i21 += 8;
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -454,8 +454,8 @@
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
i22 += 8;
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -467,8 +467,8 @@
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
i23 += 8;
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -480,8 +480,8 @@
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
i24 += 8;
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
@@ -561,14 +561,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -579,8 +579,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -591,8 +591,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -603,8 +603,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -615,8 +615,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -627,8 +627,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -639,8 +639,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -651,8 +651,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -663,8 +663,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -675,8 +675,8 @@
const __m128i vi9x01234567 = _mm_loadl_epi64((const __m128i*) i9);
const __m128i vk9x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 72 * sizeof(int8_t)));
- const __m128i vxi9x01234567 = _mm_unpacklo_epi8(vi9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi9x01234567));
- const __m128i vxk9x01234567 = _mm_unpacklo_epi8(vk9x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk9x01234567));
+ const __m128i vxi9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi9x01234567, vi9x01234567), 8);
+ const __m128i vxk9x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk9x01234567, vk9x01234567), 8);
const __m128i vp9x01234567lo = _mm_mullo_epi16(vxi9x01234567, vxk9x01234567);
const __m128i vp9x01234567hi = _mm_mulhi_epi16(vxi9x01234567, vxk9x01234567);
@@ -687,8 +687,8 @@
const __m128i vi10x01234567 = _mm_loadl_epi64((const __m128i*) i10);
const __m128i vk10x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 80 * sizeof(int8_t)));
- const __m128i vxi10x01234567 = _mm_unpacklo_epi8(vi10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi10x01234567));
- const __m128i vxk10x01234567 = _mm_unpacklo_epi8(vk10x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk10x01234567));
+ const __m128i vxi10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi10x01234567, vi10x01234567), 8);
+ const __m128i vxk10x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk10x01234567, vk10x01234567), 8);
const __m128i vp10x01234567lo = _mm_mullo_epi16(vxi10x01234567, vxk10x01234567);
const __m128i vp10x01234567hi = _mm_mulhi_epi16(vxi10x01234567, vxk10x01234567);
@@ -699,8 +699,8 @@
const __m128i vi11x01234567 = _mm_loadl_epi64((const __m128i*) i11);
const __m128i vk11x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 88 * sizeof(int8_t)));
- const __m128i vxi11x01234567 = _mm_unpacklo_epi8(vi11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi11x01234567));
- const __m128i vxk11x01234567 = _mm_unpacklo_epi8(vk11x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk11x01234567));
+ const __m128i vxi11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi11x01234567, vi11x01234567), 8);
+ const __m128i vxk11x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk11x01234567, vk11x01234567), 8);
const __m128i vp11x01234567lo = _mm_mullo_epi16(vxi11x01234567, vxk11x01234567);
const __m128i vp11x01234567hi = _mm_mulhi_epi16(vxi11x01234567, vxk11x01234567);
@@ -711,8 +711,8 @@
const __m128i vi12x01234567 = _mm_loadl_epi64((const __m128i*) i12);
const __m128i vk12x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 96 * sizeof(int8_t)));
- const __m128i vxi12x01234567 = _mm_unpacklo_epi8(vi12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi12x01234567));
- const __m128i vxk12x01234567 = _mm_unpacklo_epi8(vk12x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk12x01234567));
+ const __m128i vxi12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi12x01234567, vi12x01234567), 8);
+ const __m128i vxk12x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk12x01234567, vk12x01234567), 8);
const __m128i vp12x01234567lo = _mm_mullo_epi16(vxi12x01234567, vxk12x01234567);
const __m128i vp12x01234567hi = _mm_mulhi_epi16(vxi12x01234567, vxk12x01234567);
@@ -723,8 +723,8 @@
const __m128i vi13x01234567 = _mm_loadl_epi64((const __m128i*) i13);
const __m128i vk13x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 104 * sizeof(int8_t)));
- const __m128i vxi13x01234567 = _mm_unpacklo_epi8(vi13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi13x01234567));
- const __m128i vxk13x01234567 = _mm_unpacklo_epi8(vk13x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk13x01234567));
+ const __m128i vxi13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi13x01234567, vi13x01234567), 8);
+ const __m128i vxk13x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk13x01234567, vk13x01234567), 8);
const __m128i vp13x01234567lo = _mm_mullo_epi16(vxi13x01234567, vxk13x01234567);
const __m128i vp13x01234567hi = _mm_mulhi_epi16(vxi13x01234567, vxk13x01234567);
@@ -735,8 +735,8 @@
const __m128i vi14x01234567 = _mm_loadl_epi64((const __m128i*) i14);
const __m128i vk14x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 112 * sizeof(int8_t)));
- const __m128i vxi14x01234567 = _mm_unpacklo_epi8(vi14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi14x01234567));
- const __m128i vxk14x01234567 = _mm_unpacklo_epi8(vk14x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk14x01234567));
+ const __m128i vxi14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi14x01234567, vi14x01234567), 8);
+ const __m128i vxk14x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk14x01234567, vk14x01234567), 8);
const __m128i vp14x01234567lo = _mm_mullo_epi16(vxi14x01234567, vxk14x01234567);
const __m128i vp14x01234567hi = _mm_mulhi_epi16(vxi14x01234567, vxk14x01234567);
@@ -747,8 +747,8 @@
const __m128i vi15x01234567 = _mm_loadl_epi64((const __m128i*) i15);
const __m128i vk15x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 120 * sizeof(int8_t)));
- const __m128i vxi15x01234567 = _mm_unpacklo_epi8(vi15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi15x01234567));
- const __m128i vxk15x01234567 = _mm_unpacklo_epi8(vk15x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk15x01234567));
+ const __m128i vxi15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi15x01234567, vi15x01234567), 8);
+ const __m128i vxk15x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk15x01234567, vk15x01234567), 8);
const __m128i vp15x01234567lo = _mm_mullo_epi16(vxi15x01234567, vxk15x01234567);
const __m128i vp15x01234567hi = _mm_mulhi_epi16(vxi15x01234567, vxk15x01234567);
@@ -759,8 +759,8 @@
const __m128i vi16x01234567 = _mm_loadl_epi64((const __m128i*) i16);
const __m128i vk16x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 128 * sizeof(int8_t)));
- const __m128i vxi16x01234567 = _mm_unpacklo_epi8(vi16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi16x01234567));
- const __m128i vxk16x01234567 = _mm_unpacklo_epi8(vk16x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk16x01234567));
+ const __m128i vxi16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi16x01234567, vi16x01234567), 8);
+ const __m128i vxk16x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk16x01234567, vk16x01234567), 8);
const __m128i vp16x01234567lo = _mm_mullo_epi16(vxi16x01234567, vxk16x01234567);
const __m128i vp16x01234567hi = _mm_mulhi_epi16(vxi16x01234567, vxk16x01234567);
@@ -771,8 +771,8 @@
const __m128i vi17x01234567 = _mm_loadl_epi64((const __m128i*) i17);
const __m128i vk17x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 136 * sizeof(int8_t)));
- const __m128i vxi17x01234567 = _mm_unpacklo_epi8(vi17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi17x01234567));
- const __m128i vxk17x01234567 = _mm_unpacklo_epi8(vk17x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk17x01234567));
+ const __m128i vxi17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi17x01234567, vi17x01234567), 8);
+ const __m128i vxk17x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk17x01234567, vk17x01234567), 8);
const __m128i vp17x01234567lo = _mm_mullo_epi16(vxi17x01234567, vxk17x01234567);
const __m128i vp17x01234567hi = _mm_mulhi_epi16(vxi17x01234567, vxk17x01234567);
@@ -783,8 +783,8 @@
const __m128i vi18x01234567 = _mm_loadl_epi64((const __m128i*) i18);
const __m128i vk18x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
- const __m128i vxi18x01234567 = _mm_unpacklo_epi8(vi18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi18x01234567));
- const __m128i vxk18x01234567 = _mm_unpacklo_epi8(vk18x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk18x01234567));
+ const __m128i vxi18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi18x01234567, vi18x01234567), 8);
+ const __m128i vxk18x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk18x01234567, vk18x01234567), 8);
const __m128i vp18x01234567lo = _mm_mullo_epi16(vxi18x01234567, vxk18x01234567);
const __m128i vp18x01234567hi = _mm_mulhi_epi16(vxi18x01234567, vxk18x01234567);
@@ -795,8 +795,8 @@
const __m128i vi19x01234567 = _mm_loadl_epi64((const __m128i*) i19);
const __m128i vk19x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 152 * sizeof(int8_t)));
- const __m128i vxi19x01234567 = _mm_unpacklo_epi8(vi19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi19x01234567));
- const __m128i vxk19x01234567 = _mm_unpacklo_epi8(vk19x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk19x01234567));
+ const __m128i vxi19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi19x01234567, vi19x01234567), 8);
+ const __m128i vxk19x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk19x01234567, vk19x01234567), 8);
const __m128i vp19x01234567lo = _mm_mullo_epi16(vxi19x01234567, vxk19x01234567);
const __m128i vp19x01234567hi = _mm_mulhi_epi16(vxi19x01234567, vxk19x01234567);
@@ -807,8 +807,8 @@
const __m128i vi20x01234567 = _mm_loadl_epi64((const __m128i*) i20);
const __m128i vk20x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 160 * sizeof(int8_t)));
- const __m128i vxi20x01234567 = _mm_unpacklo_epi8(vi20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi20x01234567));
- const __m128i vxk20x01234567 = _mm_unpacklo_epi8(vk20x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk20x01234567));
+ const __m128i vxi20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi20x01234567, vi20x01234567), 8);
+ const __m128i vxk20x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk20x01234567, vk20x01234567), 8);
const __m128i vp20x01234567lo = _mm_mullo_epi16(vxi20x01234567, vxk20x01234567);
const __m128i vp20x01234567hi = _mm_mulhi_epi16(vxi20x01234567, vxk20x01234567);
@@ -819,8 +819,8 @@
const __m128i vi21x01234567 = _mm_loadl_epi64((const __m128i*) i21);
const __m128i vk21x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 168 * sizeof(int8_t)));
- const __m128i vxi21x01234567 = _mm_unpacklo_epi8(vi21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi21x01234567));
- const __m128i vxk21x01234567 = _mm_unpacklo_epi8(vk21x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk21x01234567));
+ const __m128i vxi21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi21x01234567, vi21x01234567), 8);
+ const __m128i vxk21x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk21x01234567, vk21x01234567), 8);
const __m128i vp21x01234567lo = _mm_mullo_epi16(vxi21x01234567, vxk21x01234567);
const __m128i vp21x01234567hi = _mm_mulhi_epi16(vxi21x01234567, vxk21x01234567);
@@ -831,8 +831,8 @@
const __m128i vi22x01234567 = _mm_loadl_epi64((const __m128i*) i22);
const __m128i vk22x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 176 * sizeof(int8_t)));
- const __m128i vxi22x01234567 = _mm_unpacklo_epi8(vi22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi22x01234567));
- const __m128i vxk22x01234567 = _mm_unpacklo_epi8(vk22x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk22x01234567));
+ const __m128i vxi22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi22x01234567, vi22x01234567), 8);
+ const __m128i vxk22x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk22x01234567, vk22x01234567), 8);
const __m128i vp22x01234567lo = _mm_mullo_epi16(vxi22x01234567, vxk22x01234567);
const __m128i vp22x01234567hi = _mm_mulhi_epi16(vxi22x01234567, vxk22x01234567);
@@ -843,8 +843,8 @@
const __m128i vi23x01234567 = _mm_loadl_epi64((const __m128i*) i23);
const __m128i vk23x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 184 * sizeof(int8_t)));
- const __m128i vxi23x01234567 = _mm_unpacklo_epi8(vi23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi23x01234567));
- const __m128i vxk23x01234567 = _mm_unpacklo_epi8(vk23x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk23x01234567));
+ const __m128i vxi23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi23x01234567, vi23x01234567), 8);
+ const __m128i vxk23x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk23x01234567, vk23x01234567), 8);
const __m128i vp23x01234567lo = _mm_mullo_epi16(vxi23x01234567, vxk23x01234567);
const __m128i vp23x01234567hi = _mm_mulhi_epi16(vxi23x01234567, vxk23x01234567);
@@ -855,8 +855,8 @@
const __m128i vi24x01234567 = _mm_loadl_epi64((const __m128i*) i24);
const __m128i vk24x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 192 * sizeof(int8_t)));
- const __m128i vxi24x01234567 = _mm_unpacklo_epi8(vi24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi24x01234567));
- const __m128i vxk24x01234567 = _mm_unpacklo_epi8(vk24x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk24x01234567));
+ const __m128i vxi24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi24x01234567, vi24x01234567), 8);
+ const __m128i vxk24x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk24x01234567, vk24x01234567), 8);
const __m128i vp24x01234567lo = _mm_mullo_epi16(vxi24x01234567, vxk24x01234567);
const __m128i vp24x01234567hi = _mm_mulhi_epi16(vxi24x01234567, vxk24x01234567);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c
index e7660aa..e1b3c08 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-xop-mul32.c
@@ -167,7 +167,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -441,7 +441,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -620,7 +620,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c
index 33a27ba..872b1a8 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul16.c
@@ -81,7 +81,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -231,7 +231,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c
index df62b26..1f410d2 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-avx-mul32.c
@@ -82,7 +82,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -191,7 +191,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -256,7 +256,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c
index e026153..df91eda 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse2-mul16.c
@@ -81,15 +81,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -101,8 +101,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -114,8 +114,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -127,8 +127,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -140,8 +140,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -153,8 +153,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -166,8 +166,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -179,8 +179,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -192,8 +192,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -231,14 +231,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -249,8 +249,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -261,8 +261,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -273,8 +273,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -285,8 +285,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -297,8 +297,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -309,8 +309,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -321,8 +321,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -333,8 +333,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c
index 4f35f97..08758fb 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul16.c
@@ -81,7 +81,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -231,7 +231,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c
index 28bae27..7d38698 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-sse41-mul32.c
@@ -82,7 +82,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -191,7 +191,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -256,7 +256,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c
index e703e1b..75e872a 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-fp32-xop-mul32.c
@@ -87,7 +87,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -196,7 +196,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -261,7 +261,7 @@
vscaled0123 = _mm_mul_ps(vscaled0123, _mm_load_ps(params->fp32_sse4.scale));
vacc0123 = _mm_cvtps_epi32(vscaled0123);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c
index 42dd244..f4d3e72 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul16.c
@@ -81,7 +81,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -252,7 +252,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c
index d49ff31..64b51ef 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-avx-mul32.c
@@ -82,7 +82,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -212,7 +212,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -295,7 +295,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c
index 934e8bf..1a43ae2 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse2-mul16.c
@@ -81,15 +81,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -101,8 +101,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -114,8 +114,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -127,8 +127,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -140,8 +140,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -153,8 +153,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -166,8 +166,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -179,8 +179,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -192,8 +192,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -273,14 +273,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -291,8 +291,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -303,8 +303,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -315,8 +315,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -327,8 +327,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -339,8 +339,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -351,8 +351,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -363,8 +363,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -375,8 +375,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c
index 7cba52f..f7e2d52 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul16.c
@@ -81,7 +81,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
@@ -252,7 +252,7 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c
index ddd47bb..d9c43ad 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-sse41-mul32.c
@@ -82,7 +82,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -212,7 +212,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -295,7 +295,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c
index 3ca8015..d8be7c3 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-ssse3-mul16.c
@@ -81,15 +81,15 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
i0 += 8;
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -101,8 +101,8 @@
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
i1 += 8;
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -114,8 +114,8 @@
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
i2 += 8;
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -127,8 +127,8 @@
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
i3 += 8;
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -140,8 +140,8 @@
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
i4 += 8;
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -153,8 +153,8 @@
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
i5 += 8;
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -166,8 +166,8 @@
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
i6 += 8;
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -179,8 +179,8 @@
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
i7 += 8;
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -192,8 +192,8 @@
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
i8 += 8;
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
@@ -273,14 +273,14 @@
if XNN_UNLIKELY(c != 0) {
{
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
- const __m128i vxi0x01234567 = _mm_unpacklo_epi8(vi0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi0x01234567));
- const __m128i vxk0x01234567 = _mm_unpacklo_epi8(vk0x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk0x01234567));
+ const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
+ const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
const __m128i vp0x01234567lo = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
const __m128i vp0x01234567hi = _mm_mulhi_epi16(vxi0x01234567, vxk0x01234567);
@@ -291,8 +291,8 @@
const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
- const __m128i vxi1x01234567 = _mm_unpacklo_epi8(vi1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi1x01234567));
- const __m128i vxk1x01234567 = _mm_unpacklo_epi8(vk1x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk1x01234567));
+ const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
+ const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
const __m128i vp1x01234567lo = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
const __m128i vp1x01234567hi = _mm_mulhi_epi16(vxi1x01234567, vxk1x01234567);
@@ -303,8 +303,8 @@
const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
- const __m128i vxi2x01234567 = _mm_unpacklo_epi8(vi2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi2x01234567));
- const __m128i vxk2x01234567 = _mm_unpacklo_epi8(vk2x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk2x01234567));
+ const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
+ const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
const __m128i vp2x01234567lo = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
const __m128i vp2x01234567hi = _mm_mulhi_epi16(vxi2x01234567, vxk2x01234567);
@@ -315,8 +315,8 @@
const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
- const __m128i vxi3x01234567 = _mm_unpacklo_epi8(vi3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi3x01234567));
- const __m128i vxk3x01234567 = _mm_unpacklo_epi8(vk3x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk3x01234567));
+ const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
+ const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
const __m128i vp3x01234567lo = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
const __m128i vp3x01234567hi = _mm_mulhi_epi16(vxi3x01234567, vxk3x01234567);
@@ -327,8 +327,8 @@
const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
- const __m128i vxi4x01234567 = _mm_unpacklo_epi8(vi4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi4x01234567));
- const __m128i vxk4x01234567 = _mm_unpacklo_epi8(vk4x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk4x01234567));
+ const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
+ const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
const __m128i vp4x01234567lo = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
const __m128i vp4x01234567hi = _mm_mulhi_epi16(vxi4x01234567, vxk4x01234567);
@@ -339,8 +339,8 @@
const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
- const __m128i vxi5x01234567 = _mm_unpacklo_epi8(vi5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi5x01234567));
- const __m128i vxk5x01234567 = _mm_unpacklo_epi8(vk5x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk5x01234567));
+ const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
+ const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
const __m128i vp5x01234567lo = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
const __m128i vp5x01234567hi = _mm_mulhi_epi16(vxi5x01234567, vxk5x01234567);
@@ -351,8 +351,8 @@
const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
- const __m128i vxi6x01234567 = _mm_unpacklo_epi8(vi6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi6x01234567));
- const __m128i vxk6x01234567 = _mm_unpacklo_epi8(vk6x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk6x01234567));
+ const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
+ const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
const __m128i vp6x01234567lo = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
const __m128i vp6x01234567hi = _mm_mulhi_epi16(vxi6x01234567, vxk6x01234567);
@@ -363,8 +363,8 @@
const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
- const __m128i vxi7x01234567 = _mm_unpacklo_epi8(vi7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi7x01234567));
- const __m128i vxk7x01234567 = _mm_unpacklo_epi8(vk7x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk7x01234567));
+ const __m128i vxi7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi7x01234567, vi7x01234567), 8);
+ const __m128i vxk7x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk7x01234567, vk7x01234567), 8);
const __m128i vp7x01234567lo = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
const __m128i vp7x01234567hi = _mm_mulhi_epi16(vxi7x01234567, vxk7x01234567);
@@ -375,8 +375,8 @@
const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t)));
- const __m128i vxi8x01234567 = _mm_unpacklo_epi8(vi8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vi8x01234567));
- const __m128i vxk8x01234567 = _mm_unpacklo_epi8(vk8x01234567, _mm_cmpgt_epi8(_mm_setzero_si128(), vk8x01234567));
+ const __m128i vxi8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi8x01234567, vi8x01234567), 8);
+ const __m128i vxk8x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk8x01234567, vk8x01234567), 8);
const __m128i vp8x01234567lo = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
const __m128i vp8x01234567hi = _mm_mulhi_epi16(vxi8x01234567, vxk8x01234567);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c
index 1af3502..a4ea58a 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-xop-mul32.c
@@ -87,7 +87,7 @@
const void* w = weights;
for (; c >= 8; c -= 8) {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_loadu_si32(i0));
@@ -217,7 +217,7 @@
output += 8;
}
if XNN_UNLIKELY(c != 0) {
- const int8_t* k = (const int8_t*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 8);
do {
__m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
@@ -300,7 +300,7 @@
vacc0123 =
_mm_sub_epi32(_mm_sra_epi32(vq31prod0123, vshift), _mm_cmpgt_epi32(vrem0123, vremainder_threshold));
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->gemmlowp_sse4.output_zero_point);
__m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
diff --git a/src/qs8-dwconv/unipass-sse-mul16.c.in b/src/qs8-dwconv/unipass-sse-mul16.c.in
index 94bc99c..1a1dfa0 100644
--- a/src/qs8-dwconv/unipass-sse-mul16.c.in
+++ b/src/qs8-dwconv/unipass-sse-mul16.c.in
@@ -53,7 +53,7 @@
for (; c >= ${CHANNEL_TILE}; c -= ${CHANNEL_TILE}) {
__m128i vacc${ABC[0:4]} = _mm_loadu_si128((const __m128i*) w);
$for C in range(4, CHANNEL_TILE, 4):
- __m128i vacc${ABC[C:C+4]} = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + ${C} * sizeof(int32_t)));
+ __m128i vacc${ABC[C:C+4]} = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + ${C}));
$for K in range(KERNEL_TILE):
@@ -71,8 +71,8 @@
$if SSE < 4:
$for C in range(0, CHANNEL_TILE, 8):
- const __m128i vxi${K}x${ABC[C:C+8]} = _mm_unpacklo_epi8(vi${K}x${ABC[C:C+8]}, _mm_cmpgt_epi8(_mm_setzero_si128(), vi${K}x${ABC[C:C+8]}));
- const __m128i vxk${K}x${ABC[C:C+8]} = _mm_unpacklo_epi8(vk${K}x${ABC[C:C+8]}, _mm_cmpgt_epi8(_mm_setzero_si128(), vk${K}x${ABC[C:C+8]}));
+ const __m128i vxi${K}x${ABC[C:C+8]} = _mm_srai_epi16(_mm_unpacklo_epi8(vi${K}x${ABC[C:C+8]}, vi${K}x${ABC[C:C+8]}), 8);
+ const __m128i vxk${K}x${ABC[C:C+8]} = _mm_srai_epi16(_mm_unpacklo_epi8(vk${K}x${ABC[C:C+8]}, vk${K}x${ABC[C:C+8]}), 8);
$for C in range(0, CHANNEL_TILE, 8):
const __m128i vp${K}x${ABC[C:C+8]}lo = _mm_mullo_epi16(vxi${K}x${ABC[C:C+8]}, vxk${K}x${ABC[C:C+8]});
@@ -151,8 +151,8 @@
$if DATATYPE == "QC8":
const __m128 vscale${ABC[0:4]} = _mm_loadu_ps((const float*) w);
$for C in range(4, CHANNEL_TILE, 4):
- const __m128 vscale${ABC[C:C+4]} = _mm_loadu_ps((const float*) ((uintptr_t) w + ${C} * sizeof(float)));
- w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(float));
+ const __m128 vscale${ABC[C:C+4]} = _mm_loadu_ps((const float*) w + ${C});
+ w = (const void*) ((const float*) w + ${CHANNEL_TILE});
$for C in range(0, CHANNEL_TILE, 4):
vscaled${ABC[C:C+4]} = _mm_mul_ps(vscaled${ABC[C:C+4]}, vscale${ABC[C:C+4]});
$else:
@@ -210,10 +210,10 @@
}
if XNN_UNLIKELY(c != 0) {
$if CHANNEL_TILE > 8:
- const int8_t* k = (const int8_t*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + ${CHANNEL_TILE});
${"do " if CHANNEL_TILE > 8 else ""}{
__m128i vacc${ABC[0:4]} = _mm_loadu_si128((const __m128i*) w);
- __m128i vacc${ABC[4:8]} = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 4 * sizeof(int32_t)));
+ __m128i vacc${ABC[4:8]} = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
$for K in range(KERNEL_TILE):
@@ -233,8 +233,8 @@
i${K} += 8;
$if SSE < 4:
- const __m128i vxi${K}x${ABC[0:8]} = _mm_unpacklo_epi8(vi${K}x${ABC[0:8]}, _mm_cmpgt_epi8(_mm_setzero_si128(), vi${K}x${ABC[0:8]}));
- const __m128i vxk${K}x${ABC[0:8]} = _mm_unpacklo_epi8(vk${K}x${ABC[0:8]}, _mm_cmpgt_epi8(_mm_setzero_si128(), vk${K}x${ABC[0:8]}));
+ const __m128i vxi${K}x${ABC[0:8]} = _mm_srai_epi16(_mm_unpacklo_epi8(vi${K}x${ABC[0:8]}, vi${K}x${ABC[0:8]}), 8);
+ const __m128i vxk${K}x${ABC[0:8]} = _mm_srai_epi16(_mm_unpacklo_epi8(vk${K}x${ABC[0:8]}, vk${K}x${ABC[0:8]}), 8);
const __m128i vp${K}x${ABC[0:8]}lo = _mm_mullo_epi16(vxi${K}x${ABC[0:8]}, vxk${K}x${ABC[0:8]});
const __m128i vp${K}x${ABC[0:8]}hi = _mm_mulhi_epi16(vxi${K}x${ABC[0:8]}, vxk${K}x${ABC[0:8]});
@@ -338,7 +338,7 @@
vacc${ABC[4:8]} = _mm_cvtps_epi32(vscaled${ABC[4:8]});
$if CHANNEL_TILE > 8:
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 8);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point);
diff --git a/src/qs8-dwconv/unipass-sse-mul32.c.in b/src/qs8-dwconv/unipass-sse-mul32.c.in
index 763ce72..9ff896c 100644
--- a/src/qs8-dwconv/unipass-sse-mul32.c.in
+++ b/src/qs8-dwconv/unipass-sse-mul32.c.in
@@ -62,7 +62,7 @@
for (; c >= ${CHANNEL_TILE}; c -= ${CHANNEL_TILE}) {
__m128i vacc${ABC[0:4]} = _mm_loadu_si128((const __m128i*) w);
$for C in range(4, CHANNEL_TILE, 4):
- __m128i vacc${ABC[C:C+4]} = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + ${C} * sizeof(int32_t)));
+ __m128i vacc${ABC[C:C+4]} = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + ${C}));
$for K in range(KERNEL_TILE):
@@ -115,8 +115,8 @@
$if DATATYPE == "QC8":
const __m128 vscale${ABC[0:4]} = _mm_loadu_ps((const float*) w);
$for C in range(4, CHANNEL_TILE, 4):
- const __m128 vscale${ABC[C:C+4]} = _mm_loadu_ps((const float*) ((uintptr_t) w + ${C} * sizeof(float)));
- w = (const void*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(float));
+ const __m128 vscale${ABC[C:C+4]} = _mm_loadu_ps((const float*) w + ${C});
+ w = (const void*) ((const float*) w + ${CHANNEL_TILE});
$for C in range(0, CHANNEL_TILE, 4):
vscaled${ABC[C:C+4]} = _mm_mul_ps(vscaled${ABC[C:C+4]}, vscale${ABC[C:C+4]});
$else:
@@ -156,7 +156,7 @@
}
if XNN_UNLIKELY(c != 0) {
$if CHANNEL_TILE > 4:
- const int8_t* k = (const int8_t*) ((uintptr_t) w + ${CHANNEL_TILE} * sizeof(int32_t));
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + ${CHANNEL_TILE});
${"do " if CHANNEL_TILE > 4 else ""}{
__m128i vacc${ABC[0:4]} = _mm_loadu_si128((const __m128i*) w);
@@ -213,7 +213,7 @@
vacc${ABC[0:4]} = _mm_cvtps_epi32(vscaled${ABC[0:4]});
$if CHANNEL_TILE > 4:
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point);
__m128i vout${ABC[0:4]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[0:4]}), voutput_zero_point);
diff --git a/src/qs8-gemm/MRx4c2-sse.c.in b/src/qs8-gemm/MRx4c2-sse.c.in
index c4b82e4..b655f78 100644
--- a/src/qs8-gemm/MRx4c2-sse.c.in
+++ b/src/qs8-gemm/MRx4c2-sse.c.in
@@ -82,7 +82,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
$for M in range(1, MR):
__m128i vacc${M}x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -91,7 +91,7 @@
$if SSE == 4:
const __m128i vxa${M} = _mm_cvtepi8_epi16(va${M});
$else:
- const __m128i vxa${M} = _mm_unpacklo_epi8(va${M}, _mm_cmpgt_epi8(_mm_setzero_si128(), va${M}));
+ const __m128i vxa${M} = _mm_srai_epi16(_mm_unpacklo_epi8(va${M}, va${M}), 8);
a${M} += 8;
$if VARIANT == "LD128":
@@ -99,7 +99,7 @@
$if K == 0:
const __m128i vb${K}${K+1} = _mm_loadu_si128((const __m128i*) w);
$else:
- const __m128i vb${K}${K+1} = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + ${K * 8} * sizeof(int8_t)));
+ const __m128i vb${K}${K+1} = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + ${K * 8}));
const __m128i vsb${K}${K+1} = _mm_cmpgt_epi8(_mm_setzero_si128(), vb${K}${K+1});
const __m128i vxb${K} = _mm_unpacklo_epi8(vb${K}${K+1}, vsb${K}${K+1});
const __m128i vxb${K+1} = _mm_unpackhi_epi8(vb${K}${K+1}, vsb${K}${K+1});
@@ -125,16 +125,16 @@
$if K == 0:
const __m128i vb${K} = _mm_loadl_epi64((const __m128i*) w);
$else:
- const __m128i vb${K} = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + ${K * 8} * sizeof(int8_t)));
+ const __m128i vb${K} = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + ${K * 8}));
$if SSE == 4:
const __m128i vxb${K} = _mm_cvtepi8_epi16(vb${K});
$else:
- const __m128i vxb${K} = _mm_unpacklo_epi8(vb${K}, _mm_cmpgt_epi8(_mm_setzero_si128(), vb${K}));
+ const __m128i vxb${K} = _mm_srai_epi16(_mm_unpacklo_epi8(vb${K}, vb${K}), 8);
$elif VARIANT == "EXTENDED":
$if K == 0:
const __m128i vxb${K} = _mm_load_si128((const __m128i*) w);
$else:
- const __m128i vxb${K} = _mm_load_si128((const __m128i*) ((uintptr_t) w + ${K * 8} * sizeof(int16_t)));
+ const __m128i vxb${K} = _mm_load_si128((const __m128i*) ((const int16_t*) w + ${K * 8}));
$for M in range(MR):
$if XOP:
@@ -145,9 +145,9 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa${M}, _MM_SHUFFLE(${K}, ${K}, ${K}, ${K})), vxb${K}));
$if VARIANT == "EXTENDED":
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
$else:
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -156,19 +156,19 @@
$if SSE == 4:
const __m128i vxa${M} = _mm_cvtepi8_epi16(va${M});
$else:
- const __m128i vxa${M} = _mm_unpacklo_epi8(va${M}, _mm_cmpgt_epi8(_mm_setzero_si128(), va${M}));
+ const __m128i vxa${M} = _mm_srai_epi16(_mm_unpacklo_epi8(va${M}, va${M}), 8);
a${M} = (const int8_t*) ((uintptr_t) a${M} + k);
$if VARIANT == "EXTENDED":
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
$else:
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
$if SSE == 4:
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
$else:
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
$for M in range(MR):
$if XOP:
@@ -181,14 +181,14 @@
if (k > 2 * sizeof(int8_t)) {
$if VARIANT == "EXTENDED":
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
$else:
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
$if SSE == 4:
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
$else:
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
$for M in range(MR):
$if XOP:
@@ -201,14 +201,14 @@
if (k > 4 * sizeof(int8_t)) {
$if VARIANT == "EXTENDED":
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
$else:
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
$if SSE == 4:
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
$else:
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
$for M in range(MR):
$if XOP:
@@ -304,7 +304,7 @@
$if DATATYPE == "QC8":
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
$for M in range(MR):
vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale0123);
$else:
diff --git a/src/qs8-gemm/MRx4c8-sse.c.in b/src/qs8-gemm/MRx4c8-sse.c.in
index 339bd84..e02a1d6 100644
--- a/src/qs8-gemm/MRx4c8-sse.c.in
+++ b/src/qs8-gemm/MRx4c8-sse.c.in
@@ -82,7 +82,7 @@
$for M in range(1, MR):
$for N in range(4):
__m128i vacc${M}x${N} = vacc0x${N};
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -91,7 +91,7 @@
$if SSE == 4:
const __m128i vxa${M} = _mm_cvtepi8_epi16(va${M});
$else:
- const __m128i vxa${M} = _mm_unpacklo_epi8(va${M}, _mm_cmpgt_epi8(_mm_setzero_si128(), va${M}));
+ const __m128i vxa${M} = _mm_srai_epi16(_mm_unpacklo_epi8(va${M}, va${M}), 8);
a${M} += 8;
$if VARIANT == "LD128":
@@ -99,7 +99,7 @@
$if N == 0:
const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) w);
$else:
- const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((uintptr_t) w + ${N * 8} * sizeof(int8_t)));
+ const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((const int8_t*) w + ${N * 8}));
const __m128i vsb${N}${N+1} = _mm_cmpgt_epi8(_mm_setzero_si128(), vb${N}${N+1});
const __m128i vxb${N} = _mm_unpacklo_epi8(vb${N}${N+1}, vsb${N}${N+1});
const __m128i vxb${N+1} = _mm_unpackhi_epi8(vb${N}${N+1}, vsb${N}${N+1});
@@ -117,16 +117,16 @@
$if N == 0:
const __m128i vb${N} = _mm_loadl_epi64((const __m128i*) w);
$else:
- const __m128i vb${N} = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + ${N * 8} * sizeof(int8_t)));
+ const __m128i vb${N} = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + ${N * 8}));
$if SSE == 4:
const __m128i vxb${N} = _mm_cvtepi8_epi16(vb${N});
$else:
- const __m128i vxb${N} = _mm_unpacklo_epi8(vb${N}, _mm_cmpgt_epi8(_mm_setzero_si128(), vb${N}));
+ const __m128i vxb${N} = _mm_srai_epi16(_mm_unpacklo_epi8(vb${N}, vb${N}), 8);
$elif VARIANT == "EXTENDED":
$if N == 0:
const __m128i vxb${N} = _mm_load_si128((const __m128i*) w);
$else:
- const __m128i vxb${N} = _mm_load_si128((const __m128i*) ((uintptr_t) w + ${N * 8} * sizeof(int16_t)));
+ const __m128i vxb${N} = _mm_load_si128((const __m128i*) ((const int16_t*) w + ${N * 8}));
$for M in range(MR):
$if XOP:
@@ -135,9 +135,9 @@
vacc${M}x${N} = _mm_add_epi32(vacc${M}x${N}, _mm_madd_epi16(vxa${M}, vxb${N}));
$if VARIANT == "EXTENDED":
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
$else:
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
@@ -239,7 +239,7 @@
$if DATATYPE == "QC8":
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
$for M in range(MR):
vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale0123);
$else:
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
index 6f94966..09f74fb 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,7 +72,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
index d170ad2..fe3ebcc 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -56,23 +56,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
index 4b6d42a..fac8d30 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -43,12 +43,12 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,33 +72,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
index 8d2f47b..4a9ca5c 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -43,62 +43,62 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
index 5f6cae8..4fedbb9 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,7 +72,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
index 199412c..58ee439 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -56,23 +56,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
index 4715cf7..7f34c7f 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -48,7 +48,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -66,7 +66,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -77,7 +77,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -87,7 +87,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -95,7 +95,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -103,7 +103,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
index 881d62d..0e89ca5 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -48,7 +48,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,23 +61,23 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -87,7 +87,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -95,7 +95,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -103,7 +103,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
index 8836720..23d15dc 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,7 +72,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
index 2129bc1..7ba0084 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -56,23 +56,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
index 0abda50..5ae5fb1 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -43,12 +43,12 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,33 +72,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
index 4d7ce74..6054c56 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -43,62 +43,62 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
index b4a9b0c..a8735ad 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,7 +72,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
index ffb00ef..0f68b75 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -56,23 +56,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,7 +82,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -90,7 +90,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -98,7 +98,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
index 38464ee..d30ba65 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -43,12 +43,12 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -61,7 +61,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -72,33 +72,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
index fc75b15..576491b 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -43,62 +43,62 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
index 8c30df8..8413264 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
@@ -48,7 +48,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -66,7 +66,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -77,7 +77,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -87,7 +87,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -95,7 +95,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -103,7 +103,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
index 6d69274..ef8c317 100644
--- a/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
@@ -48,7 +48,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -61,23 +61,23 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -87,7 +87,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -95,7 +95,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -103,7 +103,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
index cccabb5..0d472bc 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-avx.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -55,20 +55,20 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -77,21 +77,21 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
index f2d23c2..f979507 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse2.c
@@ -43,55 +43,55 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
index ab4b779..cbe2dc5 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-sse41.c
@@ -43,7 +43,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -55,20 +55,20 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -77,21 +77,21 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
index b56e936..2b18295 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -43,55 +43,55 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
index f756875..ce70471 100644
--- a/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/1x4c2-xw-minmax-gemmlowp-xop.c
@@ -48,7 +48,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -60,20 +60,20 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -82,21 +82,21 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
index ed48fc0..0ece629 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
index da63466..201a49e 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -57,20 +57,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
index da10e26..3036281 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -45,12 +45,12 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
index 4b1bc7f..51b7f20 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -45,32 +45,32 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
index 4009af4..807831d 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
index 7a806d0..c0fb59f 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -57,20 +57,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
index ad24d67..9900974 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
@@ -45,12 +45,12 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
index 641d3dd..2be2b09 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
@@ -45,32 +45,32 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
index 5c20f88..be748d0 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -50,7 +50,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -65,7 +65,7 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -73,7 +73,7 @@
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
index 9860e7b..f8730d0 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -50,7 +50,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -62,20 +62,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
index 993b2e4..e328c28 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
index c6f6930..c7d811e 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -57,20 +57,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
index 182a674..d10dc77 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -45,12 +45,12 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
index 3418021..4b75667 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -45,32 +45,32 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
index 8d72bbb..86efa19 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
index e63746f..8a4b07a 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -57,20 +57,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
index a54757a..9f1d1e2 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -45,12 +45,12 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -60,7 +60,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -68,7 +68,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
index bf5e58a..e2a0f6c 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -45,32 +45,32 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
index 0809765..9fb39fc 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
@@ -50,7 +50,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -65,7 +65,7 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -73,7 +73,7 @@
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
index f6399da..29f15b4 100644
--- a/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
@@ -50,7 +50,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -62,20 +62,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
index 3175475..ffadfbe 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-avx.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -56,17 +56,17 @@
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
index ddd443e..0ceef7f 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse2.c
@@ -45,28 +45,28 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
index 1e07d0d..e0f9722 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-sse41.c
@@ -45,7 +45,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -56,17 +56,17 @@
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
index 04ec0c9..258f3a8 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -45,28 +45,28 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
index 674b44b..e59b00e 100644
--- a/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/1x4c8-xw-minmax-gemmlowp-xop.c
@@ -50,7 +50,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -61,17 +61,17 @@
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
index 2516513..b6700d5 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
index 7a09d5a..46a5d8a 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -68,21 +68,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
index 7fbfbad..35ca7ab 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -50,15 +50,15 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,20 +90,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
index 0f691a5..268d7ba 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -50,60 +50,60 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
index c7f020e..bef9f23 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
index e88bd8f..f493e3d 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -68,21 +68,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
index dfe624b..7f0b000 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -95,7 +95,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -108,7 +108,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -118,7 +118,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -128,7 +128,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
index afd0f7d..7c83331 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -73,21 +73,21 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -95,7 +95,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -108,7 +108,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -118,7 +118,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -128,7 +128,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
index 27938c0..a100e20 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
index fb564ff..5af0fe1 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -68,21 +68,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
index 1c2fc39..6e2428a 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -50,15 +50,15 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,20 +90,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
index 6e2366c..7d30307 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -50,60 +50,60 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
index 417b08a..2aaebb6 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
index 0171f45..7d3030f 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -68,21 +68,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -90,7 +90,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -113,7 +113,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -123,7 +123,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
index 37bb94c..77d47a2 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -50,15 +50,15 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,20 +90,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
index f4a4f3a..336cf15 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -50,60 +50,60 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -112,8 +112,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -122,8 +122,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
index 84465b6..b42385c 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -95,7 +95,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -108,7 +108,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -118,7 +118,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -128,7 +128,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
index 7ebabff..3871cae 100644
--- a/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -73,21 +73,21 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -95,7 +95,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -108,7 +108,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -118,7 +118,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -128,7 +128,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
index df31557..217ac57 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-avx.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -67,26 +67,26 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -98,7 +98,7 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -107,7 +107,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -116,7 +116,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
index b834777..f597fe4 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse2.c
@@ -50,15 +50,15 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -67,38 +67,38 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -107,7 +107,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -116,7 +116,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
index 0aa0edd..01cb877 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-sse41.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -67,26 +67,26 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -98,7 +98,7 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -107,7 +107,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -116,7 +116,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
index 56958e0..fa1f77b 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -50,15 +50,15 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -67,38 +67,38 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -107,7 +107,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -116,7 +116,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
index 372a2b8..19bc947 100644
--- a/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/2x4c2-xw-minmax-gemmlowp-xop.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -72,26 +72,26 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -103,7 +103,7 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -112,7 +112,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -121,7 +121,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
index 791cbd4..30e816d 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
index ac2c116..1de0119 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -71,23 +71,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
index 3326d1e..3aac3b2 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -55,15 +55,15 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
index 5dc8d44..4958f6c 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -55,39 +55,39 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
index 69ad9c2..108775e 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
index 3d02fa6..f77fed1 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -71,23 +71,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
index bff063f..21cfc72 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
@@ -55,15 +55,15 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
index 43a1e38..d78a28f 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
@@ -55,39 +55,39 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
index eeeb805..e0d0e16 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -80,7 +80,7 @@
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
index 53259fa..f4751f4 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -76,23 +76,23 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
index df5de10..493d47c 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
index ae85181..053a146 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -71,23 +71,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
index 85f2c10..15709ee 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -55,15 +55,15 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
index 9813058..13a1790 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -55,39 +55,39 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
index 1691be1..1f25285 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
index c619ae7..4050bd5 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -71,23 +71,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
index bd7bd88..1d16fcf 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -55,15 +55,15 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -75,7 +75,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -85,7 +85,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
index 2928523..c3c8c15 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -55,39 +55,39 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
index 6903e30..305c931 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -80,7 +80,7 @@
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -90,7 +90,7 @@
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
index c38e481..e4058c5 100644
--- a/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -76,23 +76,23 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
index 86f9c1d..3b91043 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-avx.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -70,20 +70,20 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
index 8b67fcd..c27f102 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse2.c
@@ -55,35 +55,35 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
index ea4d124..89429b2 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-sse41.c
@@ -55,7 +55,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -70,20 +70,20 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
index 54d9ab4..cfdd6be 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -55,35 +55,35 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
index a22b257..55514b6 100644
--- a/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/2x4c8-xw-minmax-gemmlowp-xop.c
@@ -60,7 +60,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -75,20 +75,20 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
index 6081a4f..7f80b43 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
index 675b191..6eecce2 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -98,7 +98,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
index e3bdff8..d5206b1 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -57,18 +57,18 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
index 766bcc4..03c5f0e 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -57,22 +57,22 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -80,8 +80,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -89,8 +89,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -98,8 +98,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
index 88bd56d..6ee0312 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
index 2d1bad6..afab4c3 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -98,7 +98,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
index 5b4d68d..31fbe9b 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -62,7 +62,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -94,7 +94,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -113,7 +113,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -129,7 +129,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -141,7 +141,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -153,7 +153,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
index 77fcc41..b9ca245 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -62,7 +62,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -85,7 +85,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -94,7 +94,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -103,7 +103,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -113,7 +113,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -129,7 +129,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -141,7 +141,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -153,7 +153,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
index b185bd7..1cec4fd 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
index db7f469..a569469 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -98,7 +98,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
index 797d5f5..92eead7 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -57,18 +57,18 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
index 39fa2da..d24f297 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -57,22 +57,22 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -80,8 +80,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -89,8 +89,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -98,8 +98,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
index a6d3751..16ac9cc 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
index 976826e..66a2622 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -80,7 +80,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -98,7 +98,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -108,7 +108,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -136,7 +136,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -148,7 +148,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
index c6adbd4..8a87b06 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -57,18 +57,18 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -89,7 +89,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
index bf5faf3..11b6d60 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -57,22 +57,22 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -80,8 +80,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -89,8 +89,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -98,8 +98,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -108,23 +108,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -135,8 +135,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -147,8 +147,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
index 1941a8d..3b43e36 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
@@ -62,7 +62,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -94,7 +94,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -113,7 +113,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -129,7 +129,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -141,7 +141,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -153,7 +153,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
index 378abc7..4208b98 100644
--- a/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
@@ -62,7 +62,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -85,7 +85,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -94,7 +94,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -103,7 +103,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -113,7 +113,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -129,7 +129,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -141,7 +141,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -153,7 +153,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
index 48bea9e..2f95444 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-avx.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -79,7 +79,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -95,7 +95,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -104,7 +104,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,7 +119,7 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -130,7 +130,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -141,7 +141,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
index 8af9bb3..25d9d51 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse2.c
@@ -57,18 +57,18 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -79,7 +79,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -95,7 +95,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -104,22 +104,22 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -130,7 +130,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -141,7 +141,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
index 7c1fc2f..1889b5b 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-sse41.c
@@ -57,7 +57,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -79,7 +79,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -95,7 +95,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -104,7 +104,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,7 +119,7 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -130,7 +130,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -141,7 +141,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
index 4f606f2..a5e7f12 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -57,18 +57,18 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -79,7 +79,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -95,7 +95,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -104,22 +104,22 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -130,7 +130,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -141,7 +141,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
index 7d5ddb7..e9bf737 100644
--- a/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/3x4c2-xw-minmax-gemmlowp-xop.c
@@ -62,7 +62,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -84,7 +84,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -92,7 +92,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -100,7 +100,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
@@ -109,7 +109,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -124,7 +124,7 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -135,7 +135,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -146,7 +146,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
index 167c725..51ced97 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
index 1a54b46..14080c3 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -85,26 +85,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
index d0c3877..8b48c2f 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -65,18 +65,18 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
index 3bc4181..bb50736 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -65,46 +65,46 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
index dd4a934..adf2154 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
index 6fe9768..914e429 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -85,26 +85,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
index 0c042d1..4b4bcce 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
@@ -65,18 +65,18 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
index 6ae7371..48f9034 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
@@ -65,46 +65,46 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
index 4772e48..64c9089 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -70,7 +70,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -95,7 +95,7 @@
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -107,7 +107,7 @@
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
index 9dba8f5..f658d1c 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -70,7 +70,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,26 +90,26 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
index 2d31d96..97b3a83 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
index bc3e045..9e3edf6 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -85,26 +85,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
index 083b15f..ec25dbd 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -65,18 +65,18 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
index f470c48..d8ff958 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -65,46 +65,46 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
index c8d66cb..8e503ef 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
index 57b96a5..dfb509a 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -85,26 +85,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
index 148842d..8cb390d 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -65,18 +65,18 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -90,7 +90,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
index 6eb9941..831ea5c 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -65,46 +65,46 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
index c8245f1..7c308da 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
@@ -70,7 +70,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -95,7 +95,7 @@
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -107,7 +107,7 @@
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
index 560ba40..a86ce19 100644
--- a/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
@@ -70,7 +70,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -90,26 +90,26 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
index 3441a20..07de1be 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-avx.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
index 9178d3e..4e6fece 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse2.c
@@ -65,18 +65,18 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
index 1e5ef35..81d37be 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-sse41.c
@@ -65,7 +65,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
index 65119a7..c173fda 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-ssse3.c
@@ -65,18 +65,18 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
index 7a091ab..3311ccf 100644
--- a/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/3x4c8-xw-minmax-gemmlowp-xop.c
@@ -70,7 +70,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = 0;
while (k < kc) {
@@ -89,23 +89,23 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k += 8 * sizeof(int8_t);
}
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
index ea8c57a..faaf01f 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
index 34cd524..0543d49 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -92,7 +92,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -114,7 +114,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
index f399baf..387525d 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -64,21 +64,21 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
index 8c8622b..e175021 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -64,25 +64,25 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -92,8 +92,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -114,8 +114,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
index a913429..52ed1b4 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
index 842c4b8..daee210 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -92,7 +92,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -114,7 +114,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
index ca5000e..484dbfb 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -69,7 +69,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -131,7 +131,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -150,7 +150,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -164,7 +164,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -178,7 +178,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
index 6d6f815..c49adf5 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -69,7 +69,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -97,7 +97,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -119,7 +119,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -131,7 +131,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -150,7 +150,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -164,7 +164,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -178,7 +178,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
index 33d3d40..6beac29 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
index fcf7b1e..21f777b 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -92,7 +92,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -114,7 +114,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
index 13060e4..35fca32 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -64,21 +64,21 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
index 9077840..5a4a15d 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -64,25 +64,25 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -92,8 +92,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -114,8 +114,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
index 1fd19ad..62b9158 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
index bc5046e..9ea1783 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -92,7 +92,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -114,7 +114,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -126,7 +126,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -159,7 +159,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -173,7 +173,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
index a7d464f..a16334b 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -64,21 +64,21 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
index 1e11bfc..83b8adc 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -64,25 +64,25 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -92,8 +92,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -114,8 +114,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -126,26 +126,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -158,8 +158,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -172,8 +172,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
index 1b6b499..08a979a 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
@@ -69,7 +69,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -131,7 +131,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -150,7 +150,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -164,7 +164,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -178,7 +178,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
index 589bb37..a1c0442 100644
--- a/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-gemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
@@ -69,7 +69,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -97,7 +97,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -119,7 +119,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -131,7 +131,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -150,7 +150,7 @@
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -164,7 +164,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -178,7 +178,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
+ w = (const void*) ((const int8_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
index fc5d996..ffb91cb 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-avx.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -91,7 +91,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -101,7 +101,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -111,7 +111,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,7 +122,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -140,7 +140,7 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -153,7 +153,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -166,7 +166,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
index f70d278..20c8575 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse2.c
@@ -64,21 +64,21 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -91,7 +91,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -101,7 +101,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -111,7 +111,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,25 +122,25 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -153,7 +153,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -166,7 +166,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
index de4fe5a..84c4413 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-sse41.c
@@ -64,7 +64,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -91,7 +91,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -101,7 +101,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -111,7 +111,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,7 +122,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -140,7 +140,7 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -153,7 +153,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -166,7 +166,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
index 1a1877a..caf7650 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-ssse3.c
@@ -64,21 +64,21 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
@@ -91,7 +91,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -101,7 +101,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -111,7 +111,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,25 +122,25 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -153,7 +153,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -166,7 +166,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
index 3d76b26..6c0ead0 100644
--- a/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
+++ b/src/qs8-gemm/gen/4x4c2-xw-minmax-gemmlowp-xop.c
@@ -69,7 +69,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
@@ -96,7 +96,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
- const __m128i vxb1 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 8 * sizeof(int16_t)));
+ const __m128i vxb1 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 8));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -106,7 +106,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vxb2 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int16_t)));
+ const __m128i vxb2 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 16));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
@@ -116,7 +116,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
- const __m128i vxb3 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 24 * sizeof(int16_t)));
+ const __m128i vxb3 = _mm_load_si128((const __m128i*) ((const int16_t*) w + 24));
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
@@ -127,7 +127,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -145,7 +145,7 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vxb0 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -158,7 +158,7 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vxb1 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -171,7 +171,7 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vxb2 = _mm_load_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8 * sizeof(int16_t));
+ w = (const void*) ((const int16_t*) w + 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/MRx4c2-sse.c.in b/src/qs8-igemm/MRx4c2-sse.c.in
index 944dfbe..9a103a2 100644
--- a/src/qs8-igemm/MRx4c2-sse.c.in
+++ b/src/qs8-igemm/MRx4c2-sse.c.in
@@ -78,7 +78,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
$for M in range(1, MR):
__m128i vacc${M}x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -96,7 +96,7 @@
$if SSE == 4:
const __m128i vxa${M} = _mm_cvtepi8_epi16(va${M});
$else:
- const __m128i vxa${M} = _mm_unpacklo_epi8(va${M}, _mm_cmpgt_epi8(_mm_setzero_si128(), va${M}));
+ const __m128i vxa${M} = _mm_srai_epi16(_mm_unpacklo_epi8(va${M}, va${M}), 8);
a${M} += 8;
$if VARIANT == "LD128":
@@ -104,7 +104,7 @@
$if K == 0:
const __m128i vb${K}${K+1} = _mm_loadu_si128((const __m128i*) w);
$else:
- const __m128i vb${K}${K+1} = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + ${K * 8}));
+ const __m128i vb${K}${K+1} = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + ${K * 8}));
const __m128i vsb${K}${K+1} = _mm_cmpgt_epi8(_mm_setzero_si128(), vb${K}${K+1});
const __m128i vxb${K} = _mm_unpacklo_epi8(vb${K}${K+1}, vsb${K}${K+1});
const __m128i vxb${K+1} = _mm_unpackhi_epi8(vb${K}${K+1}, vsb${K}${K+1});
@@ -129,11 +129,11 @@
$if K == 0:
const __m128i vb${K} = _mm_loadl_epi64((const __m128i*) w);
$else:
- const __m128i vb${K} = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + ${K * 8}));
+ const __m128i vb${K} = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + ${K * 8}));
$if SSE == 4:
const __m128i vxb${K} = _mm_cvtepi8_epi16(vb${K});
$else:
- const __m128i vxb${K} = _mm_unpacklo_epi8(vb${K}, _mm_cmpgt_epi8(_mm_setzero_si128(), vb${K}));
+ const __m128i vxb${K} = _mm_srai_epi16(_mm_unpacklo_epi8(vb${K}, vb${K}), 8);
$for M in range(MR):
$if XOP:
@@ -143,7 +143,7 @@
vacc${M}x0123 = _mm_add_epi32(vacc${M}x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa${M}, _MM_SHUFFLE(${K}, ${K}, ${K}, ${K})), vxb${K}));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -152,12 +152,12 @@
$if SSE == 4:
const __m128i vxa${M} = _mm_cvtepi8_epi16(va${M});
$else:
- const __m128i vxa${M} = _mm_unpacklo_epi8(va${M}, _mm_cmpgt_epi8(_mm_setzero_si128(), va${M}));
+ const __m128i vxa${M} = _mm_srai_epi16(_mm_unpacklo_epi8(va${M}, va${M}), 8);
a${M} = (const int8_t*) ((uintptr_t) a${M} + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
$for M in range(MR):
$if XOP:
@@ -169,8 +169,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
$for M in range(MR):
$if XOP:
@@ -182,8 +182,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
$for M in range(MR):
$if XOP:
@@ -281,7 +281,7 @@
$if DATATYPE == "QC8":
const __m128 vscale0123 = _mm_loadu_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
$for M in range(MR):
vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale0123);
$else:
diff --git a/src/qs8-igemm/MRx4c8-sse.c.in b/src/qs8-igemm/MRx4c8-sse.c.in
index 965c2a9..1364e26 100644
--- a/src/qs8-igemm/MRx4c8-sse.c.in
+++ b/src/qs8-igemm/MRx4c8-sse.c.in
@@ -79,7 +79,7 @@
$for M in range(1, MR):
$for N in range(4):
__m128i vacc${M}x${N} = vacc0x${N};
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -97,7 +97,7 @@
$if SSE == 4:
const __m128i vxa${M} = _mm_cvtepi8_epi16(va${M});
$else:
- const __m128i vxa${M} = _mm_unpacklo_epi8(va${M}, _mm_cmpgt_epi8(_mm_setzero_si128(), va${M}));
+ const __m128i vxa${M} = _mm_srai_epi16(_mm_unpacklo_epi8(va${M}, va${M}), 8);
a${M} += 8;
$if VARIANT == "LD128":
@@ -105,7 +105,7 @@
$if N == 0:
const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) w);
$else:
- const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((uintptr_t) w + ${N * 8}));
+ const __m128i vb${N}${N+1} = _mm_load_si128((const __m128i*) ((const int8_t*) w + ${N * 8}));
const __m128i vsb${N}${N+1} = _mm_cmpgt_epi8(_mm_setzero_si128(), vb${N}${N+1});
const __m128i vxb${N} = _mm_unpacklo_epi8(vb${N}${N+1}, vsb${N}${N+1});
const __m128i vxb${N+1} = _mm_unpackhi_epi8(vb${N}${N+1}, vsb${N}${N+1});
@@ -122,11 +122,11 @@
$if N == 0:
const __m128i vb${N} = _mm_loadl_epi64((const __m128i*) w);
$else:
- const __m128i vb${N} = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + ${N * 8}));
+ const __m128i vb${N} = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + ${N * 8}));
$if SSE == 4:
const __m128i vxb${N} = _mm_cvtepi8_epi16(vb${N});
$else:
- const __m128i vxb${N} = _mm_unpacklo_epi8(vb${N}, _mm_cmpgt_epi8(_mm_setzero_si128(), vb${N}));
+ const __m128i vxb${N} = _mm_srai_epi16(_mm_unpacklo_epi8(vb${N}, vb${N}), 8);
$for M in range(MR):
$if XOP:
@@ -134,7 +134,7 @@
$else:
vacc${M}x${N} = _mm_add_epi32(vacc${M}x${N}, _mm_madd_epi16(vxa${M}, vxb${N}));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= ${MR} * sizeof(void*);
@@ -238,7 +238,7 @@
$if DATATYPE == "QC8":
const __m128 vscale0123 = _mm_load_ps((const float*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(float));
+ w = (const void*) ((const float*) w + 4);
$for M in range(MR):
vscaled${M}x0123 = _mm_mul_ps(vscaled${M}x0123, vscale0123);
$else:
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
index 37bde25..2a2ede8 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,7 +82,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
index 0b84a74..4a815b9 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-avx-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -66,23 +66,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
index a5cbc1d..925cfa5 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,7 +58,7 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,33 +82,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
index 55e4b0c..d8f5d52 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse2-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,57 +58,57 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
index f9324e3..d363292 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,7 +82,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
index abed29a..43fdb55 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-sse41-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -66,23 +66,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
index f26464a..1e3d6a8 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,7 +76,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -87,7 +87,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -96,24 +96,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
index 3170471..c9cd4c0 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-fp32-xop-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,23 +71,23 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -96,24 +96,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
index 661d8ba..d84e43a 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,7 +82,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
index 7d25a0b..0c41602 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-avx-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -66,23 +66,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
index c0dcabd..28b48ca 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,7 +58,7 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,33 +82,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
index 05f8476..2b65df2 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,57 +58,57 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
index 4ea067c..1f8b868 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,7 +82,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
index 27eebfa..6f97176 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -66,23 +66,23 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -91,24 +91,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
index 7cd1c20..d006ea0 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,7 +58,7 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -82,33 +82,33 @@
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
index 4ac0e38..2d323e1 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -45,7 +45,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -58,57 +58,57 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
index 93e6feb..a8fb8da 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,7 +76,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -87,7 +87,7 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -96,24 +96,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
index 86570d8..7cdc634 100644
--- a/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c2-minmax-gemmlowp-xop-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,23 +71,23 @@
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc0x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -96,24 +96,24 @@
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
index a81f6a4..c9be575 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
index ead06e3..0568e12 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-avx-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -68,20 +68,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
index 4360fb7..6711743 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,7 +61,7 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
index b1cffaf..f16ea89 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse2-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,27 +61,27 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
index ef57058..4bd1f41 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
index dfc49bf..2a31258 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-sse41-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -68,20 +68,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
index cc9b9c2..6f85867 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,7 +61,7 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
index c19d705..db23866 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-ssse3-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,27 +61,27 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
index 406550e..3fdbf3a 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld128.c
@@ -53,7 +53,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,7 +76,7 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -84,7 +84,7 @@
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
index 2894376..6e56155 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-fp32-xop-ld64.c
@@ -53,7 +53,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,20 +73,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
index 9bad1ab..17b8f27 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
index 5c58a64..69009c7 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-avx-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -68,20 +68,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
index 8429156..ec61f42 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,7 +61,7 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
index b7a1222..50c2d49 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,27 +61,27 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
index be97585..d406f4e 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
index f5d3dcf..e2b9bef 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -68,20 +68,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
index 8e085ec..faaf3f5 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,7 +61,7 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -71,7 +71,7 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -79,7 +79,7 @@
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
index 83d2a34..b8bd4c5 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -48,7 +48,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -61,27 +61,27 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
index d9d0c90..c8d1f8e 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld128.c
@@ -53,7 +53,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,7 +76,7 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -84,7 +84,7 @@
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
index 11011b1..0d1cb95 100644
--- a/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/1x4c8-minmax-gemmlowp-xop-ld64.c
@@ -53,7 +53,7 @@
__m128i vacc0x1 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[1]);
__m128i vacc0x2 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[2]);
__m128i vacc0x3 = _mm_cvtsi32_si128((int) ((const int32_t*) w)[3]);
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,20 +73,20 @@
const __m128i vxb0 = _mm_cvtepi8_epi16(vb0);
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 1 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
index 2a6314a..6d12bd6 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
index 4ade5aa..e89696a 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-avx-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -80,21 +80,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
index 2cebe9c..b7f7f8d 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,10 +67,10 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,20 +102,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
index a373cc6..398ee81 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse2-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,55 +67,55 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
index b2232e7..791ba21 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
index 074b5d0..1b839e6 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-sse41-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -80,21 +80,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
index a53f14d..d80dda7 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld128.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -92,7 +92,7 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -107,7 +107,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,8 +119,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -129,8 +129,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -139,8 +139,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
index a5349ee..f4f8b9a 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-fp32-xop-ld64.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,21 +85,21 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -107,7 +107,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,8 +119,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -129,8 +129,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -139,8 +139,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
index 33a075a..81f4c32 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
index 2edacd9..9d164eb 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-avx-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -80,21 +80,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
index 8dada86..39aea8e 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,10 +67,10 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,20 +102,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
index 402b6d4..128ac34 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,55 +67,55 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
index cce3b5f..9245e7e 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
index 9d7733e..f97b73c 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -80,21 +80,21 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -102,7 +102,7 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -114,8 +114,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
index c91fcd9..d3c9714 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,10 +67,10 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -87,7 +87,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -102,20 +102,20 @@
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
index f790aef..bf47976 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -50,7 +50,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -67,55 +67,55 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
vacc1x0123 = _mm_add_epi32(vacc1x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -124,8 +124,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -134,8 +134,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
index 51cc3cf..aefa9c1 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld128.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -92,7 +92,7 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -107,7 +107,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,8 +119,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -129,8 +129,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -139,8 +139,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
index 251dded..f88b0f1 100644
--- a/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c2-minmax-gemmlowp-xop-ld64.c
@@ -55,7 +55,7 @@
do {
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,21 +85,21 @@
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -107,7 +107,7 @@
vacc1x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc1x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -119,8 +119,8 @@
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -129,8 +129,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -139,8 +139,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
index 00d798f..aa4745d 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
index 56827ed..26f50da 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-avx-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
index 834cbdf..e46e6ed 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,10 +73,10 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
index 8f98520..82e5115 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse2-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,34 +73,34 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
index e42f500..0cacb43 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
index de772cb..69e9ef8 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-sse41-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
index d3f20da..dc0a620 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,10 +73,10 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
index 88a422b..e8e086e 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-ssse3-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,34 +73,34 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
index 5c7af79..ada4f99 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld128.c
@@ -61,7 +61,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -93,7 +93,7 @@
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -103,7 +103,7 @@
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
index a725181..0f3b693 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-fp32-xop-ld64.c
@@ -61,7 +61,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -89,23 +89,23 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
index ceed36c..987dcb0 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
index ca30e42..3206354 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-avx-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
index bc54486..4f88e7c 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,10 +73,10 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
index c16c117..29f6bb9 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,34 +73,34 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
index cd2d9f2..fc1ce4d 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
index 6eb3515..696aea2 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -84,23 +84,23 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
index 727aac5..ce4dbf1 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,10 +73,10 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -88,7 +88,7 @@
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -98,7 +98,7 @@
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
index 5bfabbd..e7c1e34 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -56,7 +56,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -73,34 +73,34 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
index d3a2605..c690081 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld128.c
@@ -61,7 +61,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -93,7 +93,7 @@
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -103,7 +103,7 @@
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
index 717b1ae..f69db77 100644
--- a/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/2x4c8-minmax-gemmlowp-xop-ld64.c
@@ -61,7 +61,7 @@
__m128i vacc1x1 = vacc0x1;
__m128i vacc1x2 = vacc0x2;
__m128i vacc1x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -89,23 +89,23 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 2 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
index 76801ec..39e1fc4 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
index 60b05d1..d939ab6 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-avx-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -94,7 +94,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -112,7 +112,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
index 1ec2f1e..a2ecb0f 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,13 +76,13 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
index 90828cd..4642e48 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse2-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,17 +76,17 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -94,8 +94,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -112,8 +112,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
index 72b509c..020f028 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
index 0fba4e3..9e3ff00 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-sse41-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -94,7 +94,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -112,7 +112,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
index cfc480e..d14a6c9 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -127,7 +127,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -142,8 +142,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -154,8 +154,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -166,8 +166,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
index cf86f08..a2b6c23 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-fp32-xop-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -99,7 +99,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -117,7 +117,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -127,7 +127,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -142,8 +142,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -154,8 +154,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -166,8 +166,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
index 0c941bd..abed9b6 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
index e98b019..37cc36d 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-avx-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -94,7 +94,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -112,7 +112,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
index 51b1271..3836866 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,13 +76,13 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
index 25b9a16..5d1886d 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,17 +76,17 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -94,8 +94,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -112,8 +112,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
index 02850f4..1ffbff8 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
index b886943..84d280b 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -94,7 +94,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -112,7 +112,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -122,7 +122,7 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -137,8 +137,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
index c9743bc..92e9447 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,13 +76,13 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -103,7 +103,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
index 5b6f8e6..75700c3 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -55,7 +55,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -76,17 +76,17 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -94,8 +94,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -103,8 +103,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -112,8 +112,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -122,23 +122,23 @@
vacc2x0123 = _mm_add_epi32(vacc2x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -149,8 +149,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -161,8 +161,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
index 7df63c6..6356e76 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -127,7 +127,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -142,8 +142,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -154,8 +154,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -166,8 +166,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
index 197a6af..4912f55 100644
--- a/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c2-minmax-gemmlowp-xop-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc0x0123 = _mm_loadu_si128((const __m128i*) w);
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -99,7 +99,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -108,7 +108,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -117,7 +117,7 @@
_mm_shuffle_epi32(vxa1, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc1x0123);
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -127,7 +127,7 @@
vacc2x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc2x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -142,8 +142,8 @@
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -154,8 +154,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -166,8 +166,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
index 4116936..f198a7d 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
index aa33476..fb028a4 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-avx-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -100,26 +100,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
index bd5324f..8dc7fdb 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,13 +85,13 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
index 2a81f95..079ee69 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse2-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,41 +85,41 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
index 6531f50..d262753 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
index 500f75d..a3f8f88 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-sse41-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -100,26 +100,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
index 8831628..8ffc6fc 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,13 +85,13 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
index 036b585..6c712bf 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-ssse3-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,41 +85,41 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
index 883264f..55639ef 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld128.c
@@ -69,7 +69,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -110,7 +110,7 @@
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
index 439051b..19f5904 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-fp32-xop-ld64.c
@@ -69,7 +69,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,26 +105,26 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
index b49c379..a13dd17 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
index a7d9926..fbad091 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-avx-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -100,26 +100,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
index a59b40a..e4200cd 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,13 +85,13 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
index ca33a86..a5f3732 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse2-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,41 +85,41 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
index ca1f14f..15de5b5 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
index d658603..610c200 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-sse41-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -100,26 +100,26 @@
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
index 1ec5500..b5919d7 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld128.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,13 +85,13 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb01 = _mm_load_si128((const __m128i*) w);
@@ -105,7 +105,7 @@
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -117,7 +117,7 @@
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
index ebf492b..3f6204f 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-ssse3-ld64.c
@@ -64,7 +64,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,41 +85,41 @@
size_t k = 0;
while (k < kc) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0 = _mm_add_epi32(vacc0x0, _mm_madd_epi16(vxa0, vxb0));
vacc1x0 = _mm_add_epi32(vacc1x0, _mm_madd_epi16(vxa1, vxb0));
vacc2x0 = _mm_add_epi32(vacc2x0, _mm_madd_epi16(vxa2, vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x1 = _mm_add_epi32(vacc0x1, _mm_madd_epi16(vxa0, vxb1));
vacc1x1 = _mm_add_epi32(vacc1x1, _mm_madd_epi16(vxa1, vxb1));
vacc2x1 = _mm_add_epi32(vacc2x1, _mm_madd_epi16(vxa2, vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x2 = _mm_add_epi32(vacc0x2, _mm_madd_epi16(vxa0, vxb2));
vacc1x2 = _mm_add_epi32(vacc1x2, _mm_madd_epi16(vxa1, vxb2));
vacc2x2 = _mm_add_epi32(vacc2x2, _mm_madd_epi16(vxa2, vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x3 = _mm_add_epi32(vacc0x3, _mm_madd_epi16(vxa0, vxb3));
vacc1x3 = _mm_add_epi32(vacc1x3, _mm_madd_epi16(vxa1, vxb3));
vacc2x3 = _mm_add_epi32(vacc2x3, _mm_madd_epi16(vxa2, vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
index 510b01b..a866141 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld128.c
@@ -69,7 +69,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -110,7 +110,7 @@
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb23 = _mm_load_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_load_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -122,7 +122,7 @@
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
index 3c38797..c297cc2 100644
--- a/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/3x4c8-minmax-gemmlowp-xop-ld64.c
@@ -69,7 +69,7 @@
__m128i vacc2x1 = vacc0x1;
__m128i vacc2x2 = vacc0x2;
__m128i vacc2x3 = vacc0x3;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -105,26 +105,26 @@
vacc0x0 = _mm_maddd_epi16(vxa0, vxb0, vacc0x0);
vacc1x0 = _mm_maddd_epi16(vxa1, vxb0, vacc1x0);
vacc2x0 = _mm_maddd_epi16(vxa2, vxb0, vacc2x0);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x1 = _mm_maddd_epi16(vxa0, vxb1, vacc0x1);
vacc1x1 = _mm_maddd_epi16(vxa1, vxb1, vacc1x1);
vacc2x1 = _mm_maddd_epi16(vxa2, vxb1, vacc2x1);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x2 = _mm_maddd_epi16(vxa0, vxb2, vacc0x2);
vacc1x2 = _mm_maddd_epi16(vxa1, vxb2, vacc1x2);
vacc2x2 = _mm_maddd_epi16(vxa2, vxb2, vacc2x2);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x3 = _mm_maddd_epi16(vxa0, vxb3, vacc0x3);
vacc1x3 = _mm_maddd_epi16(vxa1, vxb3, vacc1x3);
vacc2x3 = _mm_maddd_epi16(vxa2, vxb3, vacc2x3);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k += 8 * sizeof(int8_t);
}
p -= 3 * sizeof(void*);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
index 20e55bd..9575adb 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
index d5e1122..86dbc98 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-avx-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -130,7 +130,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
index e8144fe..31c84c2 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,16 +85,16 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
index b305c8d..0d6a7da 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse2-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,20 +85,20 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -108,8 +108,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -119,8 +119,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -130,8 +130,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
index 48ff03e..112e243 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
index cb3166d..c8a3cca 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-sse41-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -130,7 +130,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
index 48d99be..efa3ada 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -124,7 +124,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -147,7 +147,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -165,8 +165,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -179,8 +179,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -193,8 +193,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
index 9387431..9f9ad98 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-fp32-xop-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -113,7 +113,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -124,7 +124,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -135,7 +135,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -147,7 +147,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -165,8 +165,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -179,8 +179,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -193,8 +193,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
index de5da17..681ea5f 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
index d4f6be1..7e66cc1 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-avx-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -130,7 +130,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
index 3c7af53..5875e0e 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,16 +85,16 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
index e40ce90..89ba44a 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse2-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,20 +85,20 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -108,8 +108,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -119,8 +119,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -130,8 +130,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
index e577a64..64e1ff5 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
index 2ab9bc8..8571512 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-sse41-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -108,7 +108,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -130,7 +130,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
@@ -142,7 +142,7 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -160,8 +160,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
index f844b67..35d07a0 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld128.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,16 +85,16 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb01 = _mm_loadu_si128((const __m128i*) w);
@@ -119,7 +119,7 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
index d8765d9..5d141e0 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-ssse3-ld64.c
@@ -60,7 +60,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -85,20 +85,20 @@
size_t k = kc;
while (k >= 8 * sizeof(int8_t)) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 += 8;
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 += 8;
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 += 8;
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 += 8;
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -108,8 +108,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -119,8 +119,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
@@ -130,8 +130,8 @@
_mm_madd_epi16(_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
- const __m128i vxb3 = _mm_unpacklo_epi8(vb3, _mm_cmpgt_epi8(_mm_setzero_si128(), vb3));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
+ const __m128i vxb3 = _mm_srai_epi16(_mm_unpacklo_epi8(vb3, vb3), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
@@ -142,26 +142,26 @@
vacc3x0123 = _mm_add_epi32(vacc3x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3));
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
const __m128i va0 = _mm_loadl_epi64((const __m128i*) a0);
- const __m128i vxa0 = _mm_unpacklo_epi8(va0, _mm_cmpgt_epi8(_mm_setzero_si128(), va0));
+ const __m128i vxa0 = _mm_srai_epi16(_mm_unpacklo_epi8(va0, va0), 8);
a0 = (const int8_t*) ((uintptr_t) a0 + k);
const __m128i va1 = _mm_loadl_epi64((const __m128i*) a1);
- const __m128i vxa1 = _mm_unpacklo_epi8(va1, _mm_cmpgt_epi8(_mm_setzero_si128(), va1));
+ const __m128i vxa1 = _mm_srai_epi16(_mm_unpacklo_epi8(va1, va1), 8);
a1 = (const int8_t*) ((uintptr_t) a1 + k);
const __m128i va2 = _mm_loadl_epi64((const __m128i*) a2);
- const __m128i vxa2 = _mm_unpacklo_epi8(va2, _mm_cmpgt_epi8(_mm_setzero_si128(), va2));
+ const __m128i vxa2 = _mm_srai_epi16(_mm_unpacklo_epi8(va2, va2), 8);
a2 = (const int8_t*) ((uintptr_t) a2 + k);
const __m128i va3 = _mm_loadl_epi64((const __m128i*) a3);
- const __m128i vxa3 = _mm_unpacklo_epi8(va3, _mm_cmpgt_epi8(_mm_setzero_si128(), va3));
+ const __m128i vxa3 = _mm_srai_epi16(_mm_unpacklo_epi8(va3, va3), 8);
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0));
@@ -174,8 +174,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1));
@@ -188,8 +188,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_add_epi32(vacc0x0123,
_mm_madd_epi16(_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2));
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
index 61b2117..49717e2 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld128.c
@@ -65,7 +65,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -124,7 +124,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb23 = _mm_loadu_si128((const __m128i*) ((const int8_t*) w + 16));
const __m128i vsb23 = _mm_cmpgt_epi8(_mm_setzero_si128(), vb23);
const __m128i vxb2 = _mm_unpacklo_epi8(vb23, vsb23);
const __m128i vxb3 = _mm_unpackhi_epi8(vb23, vsb23);
@@ -147,7 +147,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -165,8 +165,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -179,8 +179,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -193,8 +193,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);
diff --git a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
index fc69a9c..d4b022a 100644
--- a/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
+++ b/src/qs8-igemm/gen/4x4c2-minmax-gemmlowp-xop-ld64.c
@@ -65,7 +65,7 @@
__m128i vacc1x0123 = vacc0x0123;
__m128i vacc2x0123 = vacc0x0123;
__m128i vacc3x0123 = vacc0x0123;
- w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
+ w = (const void*) ((const int32_t*) w + 4);
size_t p = ks;
do {
@@ -113,7 +113,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc3x0123);
- const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8));
+ const __m128i vb1 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 8));
const __m128i vxb1 = _mm_cvtepi8_epi16(vb1);
vacc0x0123 = _mm_maddd_epi16(
@@ -124,7 +124,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc3x0123);
- const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16));
+ const __m128i vb2 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 16));
const __m128i vxb2 = _mm_cvtepi8_epi16(vb2);
vacc0x0123 = _mm_maddd_epi16(
@@ -135,7 +135,7 @@
_mm_shuffle_epi32(vxa2, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc2x0123);
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc3x0123);
- const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24));
+ const __m128i vb3 = _mm_loadl_epi64((const __m128i*) ((const int8_t*) w + 24));
const __m128i vxb3 = _mm_cvtepi8_epi16(vb3);
vacc0x0123 = _mm_maddd_epi16(
@@ -147,7 +147,7 @@
vacc3x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa3, _MM_SHUFFLE(3, 3, 3, 3)), vxb3, vacc3x0123);
- w = (const void*) ((uintptr_t) w + 32);
+ w = (const void*) ((const int8_t*) w + 32);
k -= 8 * sizeof(int8_t);
}
if (k != 0) {
@@ -165,8 +165,8 @@
a3 = (const int8_t*) ((uintptr_t) a3 + k);
const __m128i vb0 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb0 = _mm_unpacklo_epi8(vb0, _mm_cmpgt_epi8(_mm_setzero_si128(), vb0));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb0 = _mm_srai_epi16(_mm_unpacklo_epi8(vb0, vb0), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(0, 0, 0, 0)), vxb0, vacc0x0123);
@@ -179,8 +179,8 @@
if (k > 2 * sizeof(int8_t)) {
const __m128i vb1 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb1 = _mm_unpacklo_epi8(vb1, _mm_cmpgt_epi8(_mm_setzero_si128(), vb1));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb1 = _mm_srai_epi16(_mm_unpacklo_epi8(vb1, vb1), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(1, 1, 1, 1)), vxb1, vacc0x0123);
@@ -193,8 +193,8 @@
if (k > 4 * sizeof(int8_t)) {
const __m128i vb2 = _mm_loadl_epi64((const __m128i*) w);
- w = (const void*) ((uintptr_t) w + 8);
- const __m128i vxb2 = _mm_unpacklo_epi8(vb2, _mm_cmpgt_epi8(_mm_setzero_si128(), vb2));
+ w = (const void*) ((const int8_t*) w + 8);
+ const __m128i vxb2 = _mm_srai_epi16(_mm_unpacklo_epi8(vb2, vb2), 8);
vacc0x0123 = _mm_maddd_epi16(
_mm_shuffle_epi32(vxa0, _MM_SHUFFLE(2, 2, 2, 2)), vxb2, vacc0x0123);