src/qs8-vaddc/avx512skx-mul32-ld128.c.in

// Copyright 2021 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

$assert DATATYPE in ["QS8", "QU8"]
$assert BATCH_TILE % 16 == 0
$assert BATCH_TILE >= 16
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
#include <assert.h>

#include <immintrin.h>

#include <xnnpack/intrinsics-polyfill.h>
#include <xnnpack/vaddsub.h>


$XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE]
$_MM512_CVTEPX8_EPI32 = {"QS8": "_mm512_cvtepi8_epi32", "QU8": "_mm512_cvtepu8_epi32"}[DATATYPE]
$_MM256_PACKXS_EPI16 = {"QS8": "_mm256_packs_epi16", "QU8": "_mm256_packus_epi16"}[DATATYPE]
$_MM_PACKXS_EPI16 = {"QS8": "_mm_packs_epi16", "QU8": "_mm_packus_epi16"}[DATATYPE]
$_MM256_MIN_EPX8 = {"QS8": "_mm256_min_epi8", "QU8": "_mm256_min_epu8"}[DATATYPE]
$_MM256_MAX_EPX8 = {"QS8": "_mm256_max_epi8", "QU8": "_mm256_max_epu8"}[DATATYPE]
$_MM_MIN_EPX8 = {"QS8": "_mm_min_epi8", "QU8": "_mm_min_epu8"}[DATATYPE]
$_MM_MAX_EPX8 = {"QS8": "_mm_max_epi8", "QU8": "_mm_max_epu8"}[DATATYPE]
void xnn_${DATATYPE.lower()}_vaddc_minmax_ukernel__avx512skx_mul32_ld128_x${BATCH_TILE}(
    size_t n,
    const ${XINT8_T}* input_a,
    const ${XINT8_T}* input_b,
    ${XINT8_T}* output,
    const union xnn_${DATATYPE.lower()}_addsub_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
{
  const __m512i va_multiplier = _mm512_load_si512(params->avx512.a_multiplier);
  const __m128i vshift = _mm_loadu_si32(params->avx512.shift);
  $if BATCH_TILE > 16:
    const __m512i voutput_zero_point = _mm512_load_si512(params->avx512.output_zero_point);
    const __m256i voutput_min = _mm256_load_si256((const __m256i*) params->avx512.output_min);
    const __m256i voutput_max = _mm256_load_si256((const __m256i*) params->avx512.output_max);
  $else:
    const __m256i voutput_zero_point = _mm256_load_si256((const __m256i*) params->avx512.output_zero_point);
    const __m128i voutput_min = _mm_load_si128((const __m128i*) params->avx512.output_min);
    const __m128i voutput_max = _mm_load_si128((const __m128i*) params->avx512.output_max);

  const __m512i vbias = _mm512_add_epi32(
    _mm512_broadcastd_epi32(_mm_cvtsi32_si128(params->avx512.b_multiplier[0] * (int32_t) *input_b)),
    _mm512_load_si512(params->avx512.bias));
  for (; n >= ${BATCH_TILE} * sizeof(${XINT8_T}); n -= ${BATCH_TILE} * sizeof(${XINT8_T})) {
    const __m512i va${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) input_a));
    $for N in range(16, BATCH_TILE, 16):
      const __m512i va${ABC[N:N+16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) (input_a + ${N})));
    input_a += ${BATCH_TILE};

    $for N in range(0, BATCH_TILE, 16):
      __m512i vacc${ABC[N:N+16]} = _mm512_add_epi32(vbias, _mm512_mullo_epi32(va${ABC[N:N+16]}, va_multiplier));

    $for N in range(0, BATCH_TILE, 16):
      vacc${ABC[N:N+16]} = _mm512_sra_epi32(vacc${ABC[N:N+16]}, vshift);

    $for N in range(0, BATCH_TILE, 32):
      $if N + 16 < BATCH_TILE:
        __m512i vout${ABC[N:N+4]}${ABC[N+16:N+20]}${ABC[N+4:N+8]}${ABC[N+20:N+24]}${ABC[N+8:N+12]}${ABC[N+24:N+28]}${ABC[N+12:N+16]}${ABC[N+28:N+32]} = _mm512_adds_epi16(_mm512_packs_epi32(vacc${ABC[N:N+16]}, vacc${ABC[N+16:N+32]}), voutput_zero_point);
      $elif BATCH_TILE > 16:
        __m256i vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[N:N+16]}), _mm512_extracti32x8_epi32(vacc${ABC[N:N+16]}, 1)), _mm512_castsi512_si256(voutput_zero_point));
      $else:
        __m256i vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[N:N+16]}), _mm512_extracti32x8_epi32(vacc${ABC[N:N+16]}, 1)), voutput_zero_point);

    $for N in range(0, BATCH_TILE, 32):
      $if N + 16 < BATCH_TILE:
        __m256i vout${ABC[N:N+32]} = _mm256_permutevar8x32_epi32(${_MM256_PACKXS_EPI16}(_mm512_castsi512_si256(vout${ABC[N:N+4]}${ABC[N+16:N+20]}${ABC[N+4:N+8]}${ABC[N+20:N+24]}${ABC[N+8:N+12]}${ABC[N+24:N+28]}${ABC[N+12:N+16]}${ABC[N+28:N+32]}), _mm512_extracti32x8_epi32(vout${ABC[N:N+4]}${ABC[N+16:N+20]}${ABC[N+4:N+8]}${ABC[N+20:N+24]}${ABC[N+8:N+12]}${ABC[N+24:N+28]}${ABC[N+12:N+16]}${ABC[N+28:N+32]}, 1)), _mm256_set_epi32(7, 3, 5, 1, 6, 2, 4, 0));
      $else:
        __m128i vout${ABC[N:N+16]} = _mm_shuffle_epi32(${_MM_PACKXS_EPI16}(_mm256_castsi256_si128(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}), _mm256_extracti128_si256(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}, 1)), _MM_SHUFFLE(3, 1, 2, 0));

    $for N in range(0, BATCH_TILE, 32):
      $if N + 16 < BATCH_TILE:
        vout${ABC[N:N+32]} = ${_MM256_MAX_EPX8}(vout${ABC[N:N+32]}, voutput_min);
      $elif BATCH_TILE > 16:
        vout${ABC[N:N+16]} = ${_MM_MAX_EPX8}(vout${ABC[N:N+16]}, _mm256_castsi256_si128(voutput_min));
      $else:
        vout${ABC[N:N+16]} = ${_MM_MAX_EPX8}(vout${ABC[N:N+16]}, voutput_min);

    $for N in range(0, BATCH_TILE, 32):
      $if N + 16 < BATCH_TILE:
        vout${ABC[N:N+32]} = ${_MM256_MIN_EPX8}(vout${ABC[N:N+32]}, voutput_max);
      $elif BATCH_TILE > 16:
        vout${ABC[N:N+16]} = ${_MM_MIN_EPX8}(vout${ABC[N:N+16]}, _mm256_castsi256_si128(voutput_max));
      $else:
        vout${ABC[N:N+16]} = ${_MM_MIN_EPX8}(vout${ABC[N:N+16]}, voutput_max);

    $if BATCH_TILE >= 32:
      _mm256_storeu_si256((__m256i*) output, vout${ABC[0:32]});
    $else:
      _mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
    $for N in range(32, BATCH_TILE, 32):
      $if N + 8 < BATCH_TILE:
        _mm256_storeu_si256((__m256i*) (output + ${N}), vout${ABC[N:N+32]});
      $else:
        _mm_storeu_si128((__m128i*) (output + ${N}), vout${ABC[N:N+16]});
    output += ${BATCH_TILE};
  }
  if XNN_UNLIKELY(n != 0) {
    ${"do " if BATCH_TILE > 16 else ""}{
      $if BATCH_TILE > 16:
        const __m512i va${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_loadu_si128((const __m128i*) input_a));
        input_a += 16;
      $else:
        const __mmask16 vmask = _cvtu32_mask16((uint32_t) ((UINT32_C(1) << n) - UINT32_C(1)));
        const __m512i va${ABC[0:16]} = ${_MM512_CVTEPX8_EPI32}(_mm_maskz_loadu_epi8(vmask, input_a));

      __m512i vacc${ABC[0:16]} = _mm512_add_epi32(vbias, _mm512_mullo_epi32(va${ABC[0:16]}, va_multiplier));

      vacc${ABC[0:16]} = _mm512_sra_epi32(vacc${ABC[0:16]}, vshift);

      $if BATCH_TILE > 16:
        __m256i vout${ABC[0:4]}${ABC[8:12]}${ABC[4:8]}${ABC[12:16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[0:16]}), _mm512_extracti32x8_epi32(vacc${ABC[0:16]}, 1)), _mm512_castsi512_si256(voutput_zero_point));
      $else:
        __m256i vout${ABC[0:4]}${ABC[8:12]}${ABC[4:8]}${ABC[12:16]} = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc${ABC[0:16]}), _mm512_extracti32x8_epi32(vacc${ABC[0:16]}, 1)), voutput_zero_point);
      __m128i vout${ABC[0:16]} = _mm_shuffle_epi32(${_MM_PACKXS_EPI16}(_mm256_castsi256_si128(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}), _mm256_extracti128_si256(vout${ABC[N:N+4]}${ABC[N+8:N+12]}${ABC[N+4:N+8]}${ABC[N+12:N+16]}, 1)), _MM_SHUFFLE(3, 1, 2, 0));
      $if BATCH_TILE > 16:
        vout${ABC[0:16]} = ${_MM_MAX_EPX8}(vout${ABC[0:16]}, _mm256_castsi256_si128(voutput_min));
        vout${ABC[0:16]} = ${_MM_MIN_EPX8}(vout${ABC[0:16]}, _mm256_castsi256_si128(voutput_max));
      $else:
        vout${ABC[0:16]} = ${_MM_MAX_EPX8}(vout${ABC[0:16]}, voutput_min);
        vout${ABC[0:16]} = ${_MM_MIN_EPX8}(vout${ABC[0:16]}, voutput_max);

      $if BATCH_TILE > 16:
        if XNN_LIKELY(n >= (16 * sizeof(${XINT8_T}))) {
          _mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
          output += 16;
          n -= 16 * sizeof(${XINT8_T});
        } else {
          const __mmask16 vmask = _cvtu32_mask16((uint32_t) ((UINT32_C(1) << n) - UINT32_C(1)));
          _mm_mask_storeu_epi8(output, vmask, vout${ABC[0:16]});
          n = 0;
        }
      $else:
        _mm_mask_storeu_epi8(output, vmask, vout${ABC[0:16]});
    }${" while (n != 0);" if BATCH_TILE > 16 else ""}
  }
}