F32->QS8/QU8 CVT evaluation stubs for NEON and NEON v8
Prototype a more efficient quantization algorithm through exhaustive search
PiperOrigin-RevId: 413603996
diff --git a/BUILD.bazel b/BUILD.bazel
index 21646b4..2be94c8 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -2463,6 +2463,8 @@
"src/math/cvt-f16-f32-neon-int16.c",
"src/math/cvt-f16-f32-neon-int32.c",
"src/math/cvt-f32-f16-neon.c",
+ "src/math/cvt-f32-qs8-neon.c",
+ "src/math/cvt-f32-qu8-neon.c",
"src/math/expm1minus-neon-rr2-lut16-p3.c",
"src/math/expm1minus-neon-rr2-p6.c",
"src/math/roundd-neon-addsub.c",
@@ -3594,6 +3596,8 @@
"src/f32-qu8-vcvt/gen/vcvt-neonv8-x16.c",
"src/f32-qu8-vcvt/gen/vcvt-neonv8-x24.c",
"src/f32-qu8-vcvt/gen/vcvt-neonv8-x32.c",
+ "src/math/cvt-f32-qs8-neonv8.c",
+ "src/math/cvt-f32-qu8-neonv8.c",
"src/math/roundd-neonv8.c",
"src/math/roundne-neonv8.c",
"src/math/roundu-neonv8.c",
@@ -9328,6 +9332,28 @@
)
xnnpack_unit_test(
+ name = "f32_qs8_cvt_eval",
+ srcs = [
+ "eval/f32-qs8-cvt.cc",
+ "src/xnnpack/AlignedAllocator.h",
+ "src/xnnpack/math-stubs.h",
+ ] + MICROKERNEL_TEST_HDRS,
+ automatic = False,
+ deps = MICROKERNEL_TEST_DEPS,
+)
+
+xnnpack_unit_test(
+ name = "f32_qu8_cvt_eval",
+ srcs = [
+ "eval/f32-qu8-cvt.cc",
+ "src/xnnpack/AlignedAllocator.h",
+ "src/xnnpack/math-stubs.h",
+ ] + MICROKERNEL_TEST_HDRS,
+ automatic = False,
+ deps = MICROKERNEL_TEST_DEPS,
+)
+
+xnnpack_unit_test(
name = "f32_exp_eval",
srcs = [
"eval/f32-exp.cc",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3d250f7..8ed69d5 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1484,6 +1484,8 @@
src/math/cvt-f16-f32-neon-int16.c
src/math/cvt-f16-f32-neon-int32.c
src/math/cvt-f32-f16-neon.c
+ src/math/cvt-f32-qs8-neon.c
+ src/math/cvt-f32-qu8-neon.c
src/math/expm1minus-neon-rr2-lut16-p3.c
src/math/expm1minus-neon-rr2-p6.c
src/math/roundd-neon-addsub.c
@@ -2607,6 +2609,8 @@
src/f32-qu8-vcvt/gen/vcvt-neonv8-x16.c
src/f32-qu8-vcvt/gen/vcvt-neonv8-x24.c
src/f32-qu8-vcvt/gen/vcvt-neonv8-x32.c
+ src/math/cvt-f32-qs8-neonv8.c
+ src/math/cvt-f32-qu8-neonv8.c
src/math/roundd-neonv8.c
src/math/roundne-neonv8.c
src/math/roundu-neonv8.c
@@ -7735,6 +7739,22 @@
TARGET_INCLUDE_DIRECTORIES(f32-f16-cvt-eval PRIVATE include src)
TARGET_LINK_LIBRARIES(f32-f16-cvt-eval PRIVATE cpuinfo fp16 pthreadpool gtest gtest_main)
+ ADD_EXECUTABLE(f32-qs8-cvt-eval eval/f32-qs8-cvt.cc $<TARGET_OBJECTS:all_microkernels>)
+ SET_TARGET_PROPERTIES(f32-qs8-cvt-eval PROPERTIES
+ CXX_STANDARD 11
+ CXX_STANDARD_REQUIRED YES
+ CXX_EXTENSIONS NO)
+ TARGET_INCLUDE_DIRECTORIES(f32-qs8-cvt-eval PRIVATE include src)
+ TARGET_LINK_LIBRARIES(f32-qs8-cvt-eval PRIVATE cpuinfo fp16 pthreadpool gtest gtest_main)
+
+ ADD_EXECUTABLE(f32-qu8-cvt-eval eval/f32-qu8-cvt.cc $<TARGET_OBJECTS:all_microkernels>)
+ SET_TARGET_PROPERTIES(f32-qu8-cvt-eval PROPERTIES
+ CXX_STANDARD 11
+ CXX_STANDARD_REQUIRED YES
+ CXX_EXTENSIONS NO)
+ TARGET_INCLUDE_DIRECTORIES(f32-qu8-cvt-eval PRIVATE include src)
+ TARGET_LINK_LIBRARIES(f32-qu8-cvt-eval PRIVATE cpuinfo fp16 pthreadpool gtest gtest_main)
+
ADD_EXECUTABLE(f32-exp-eval eval/f32-exp.cc $<TARGET_OBJECTS:all_microkernels>)
SET_TARGET_PROPERTIES(f32-exp-eval PROPERTIES
CXX_STANDARD 11
diff --git a/eval/f32-qs8-cvt.cc b/eval/f32-qs8-cvt.cc
new file mode 100644
index 0000000..2188f2e
--- /dev/null
+++ b/eval/f32-qs8-cvt.cc
@@ -0,0 +1,269 @@
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <iomanip>
+#include <ios>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include <fp16.h>
+
+#include <xnnpack/AlignedAllocator.h>
+#include <xnnpack/common.h>
+#include <xnnpack/isa-checks.h>
+#include <xnnpack/math-stubs.h>
+
+
+constexpr int kBlockSize = 1024;
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(CVT__NEON, positive_normal) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) (std::numeric_limits<int8_t>::max() - zero_point));
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neon(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<int8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<int8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEON, negative_normal) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) (zero_point - std::numeric_limits<int8_t>::min()));
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neon(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<int8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<int8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEON, positive_saturation) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) (std::numeric_limits<int8_t>::max() - zero_point));
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neon(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<int8_t>::max();
+ ASSERT_EQ(reference_output, int32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEON, negative_saturation) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) (zero_point - std::numeric_limits<int8_t>::min()));
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neon(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<int8_t>::min();
+ ASSERT_EQ(reference_output, int32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(CVT__NEONV8, positive_normal) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) (std::numeric_limits<int8_t>::max() - zero_point));
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neonv8(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<int8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<int8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEONV8, negative_normal) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) (zero_point - std::numeric_limits<int8_t>::min()));
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neonv8(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<int8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<int8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEONV8, positive_saturation) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) (std::numeric_limits<int8_t>::max() - zero_point));
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neonv8(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<int8_t>::max();
+ ASSERT_EQ(reference_output, int32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEONV8, negative_saturation) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<int8_t, AlignedAllocator<int8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<int8_t>::min();
+ zero_point <= std::numeric_limits<int8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) (zero_point - std::numeric_limits<int8_t>::min()));
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qs8_cvt__neonv8(kBlockSize * sizeof(int8_t), inputs.data(), outputs.data(), int8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<int8_t>::min();
+ ASSERT_EQ(reference_output, int32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << int32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
diff --git a/eval/f32-qu8-cvt.cc b/eval/f32-qu8-cvt.cc
new file mode 100644
index 0000000..8ec3ef1
--- /dev/null
+++ b/eval/f32-qu8-cvt.cc
@@ -0,0 +1,269 @@
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <iomanip>
+#include <ios>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include <fp16.h>
+
+#include <xnnpack/AlignedAllocator.h>
+#include <xnnpack/common.h>
+#include <xnnpack/isa-checks.h>
+#include <xnnpack/math-stubs.h>
+
+
+constexpr int kBlockSize = 1024;
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(CVT__NEON, positive_normal) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) (std::numeric_limits<uint8_t>::max() - zero_point));
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neon(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<uint8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<uint8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEON, negative_normal) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) zero_point);
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neon(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<uint8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<uint8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEON, positive_saturation) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) (std::numeric_limits<uint8_t>::max() - zero_point));
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neon(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<uint8_t>::max();
+ ASSERT_EQ(reference_output, uint32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEON, negative_saturation) {
+ TEST_REQUIRES_ARM_NEON;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) zero_point);
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neon(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<uint8_t>::min();
+ ASSERT_EQ(reference_output, uint32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(CVT__NEONV8, positive_normal) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) (std::numeric_limits<uint8_t>::max() - zero_point));
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neonv8(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<uint8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<uint8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEONV8, negative_normal) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t max_input = fp32_to_bits((float) zero_point);
+ for (uint32_t n = 0; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neonv8(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ long reference_output = std::lrintf(inputs[i]) + long(zero_point);
+ if (inputs[i] >= float(std::numeric_limits<long>::max())) {
+ reference_output = std::numeric_limits<uint8_t>::max();
+ } else if (inputs[i] <= float(std::numeric_limits<long>::min())) {
+ reference_output = std::numeric_limits<uint8_t>::min();
+ }
+ ASSERT_EQ(reference_output, long(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEONV8, positive_saturation) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) (std::numeric_limits<uint8_t>::max() - zero_point));
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neonv8(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<uint8_t>::max();
+ ASSERT_EQ(reference_output, uint32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+
+ TEST(CVT__NEONV8, negative_saturation) {
+ TEST_REQUIRES_ARM_NEON_V8;
+
+ std::vector<float, AlignedAllocator<float, 64>> inputs(kBlockSize);
+ std::vector<uint8_t, AlignedAllocator<uint8_t, 64>> outputs(kBlockSize);
+ for (int32_t zero_point = std::numeric_limits<uint8_t>::min();
+ zero_point <= std::numeric_limits<uint8_t>::max();
+ zero_point++)
+ {
+ const uint32_t min_input = fp32_to_bits((float) zero_point);
+ const uint32_t max_input = UINT32_C(0x7F800000);
+ for (uint32_t n = min_input; n < max_input; n += kBlockSize) {
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ inputs[i] = fp32_from_bits(UINT32_C(0x80000000) | std::min<uint32_t>(n + i, max_input));
+ }
+ xnn_math_f32_qu8_cvt__neonv8(kBlockSize * sizeof(uint8_t), inputs.data(), outputs.data(), uint8_t(zero_point));
+ for (uint32_t i = 0; i < kBlockSize; i++) {
+ const int32_t reference_output = std::numeric_limits<uint8_t>::min();
+ ASSERT_EQ(reference_output, uint32_t(outputs[i]))
+ << "input = 0x" << std::hex << std::setw(8) << std::setfill('0') << fp32_to_bits(inputs[i])
+ << ", reference = " << std::dec << reference_output
+ << ", optimized = " << std::dec << uint32_t(outputs[i])
+ << ", zero point = " << std::dec << zero_point;
+ }
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
diff --git a/src/math/cvt-f32-qs8-neon.c b/src/math/cvt-f32-qs8-neon.c
new file mode 100644
index 0000000..d5f9397
--- /dev/null
+++ b/src/math/cvt-f32-qs8-neon.c
@@ -0,0 +1,47 @@
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/math-stubs.h>
+
+
+void xnn_math_f32_qs8_cvt__neon(
+ size_t n,
+ const float* input,
+ int8_t* output,
+ int8_t output_zero_point)
+{
+ assert(n % (8 * sizeof(int8_t)) == 0);
+
+ const int32x4_t vmin = vreinterpretq_s32_f32(vdupq_n_f32(12582912.0f - 128.0f - (float) output_zero_point));
+ const float32x4_t vfmagic = vdupq_n_f32(12582912.0f);
+ const int32x4_t vimagic = vdupq_n_s32(INT32_C(0x4B400000) - (int32_t) output_zero_point);
+ for (; n != 0; n -= 8 * sizeof(int8_t)) {
+ float32x4_t vx_lo = vld1q_f32(input); input += 4;
+ float32x4_t vx_hi = vld1q_f32(input); input += 4;
+
+ vx_lo = vaddq_f32(vx_lo, vfmagic);
+ vx_hi = vaddq_f32(vx_hi, vfmagic);
+
+ int32x4_t vy_lo = vreinterpretq_s32_f32(vx_lo);
+ int32x4_t vy_hi = vreinterpretq_s32_f32(vx_hi);
+
+ vy_lo = vmaxq_s32(vy_lo, vmin);
+ vy_hi = vmaxq_s32(vy_hi, vmin);
+
+ vy_lo = vsubq_s32(vy_lo, vimagic);
+ vy_hi = vsubq_s32(vy_hi, vimagic);
+
+ const int16x8_t vy = vcombine_s16(vqmovn_s32(vy_lo), vqmovn_s32(vy_hi));
+
+ const int8x8_t vout = vqmovn_s16(vy);
+ vst1_s8(output, vout); output += 8;
+ }
+}
diff --git a/src/math/cvt-f32-qs8-neonv8.c b/src/math/cvt-f32-qs8-neonv8.c
new file mode 100644
index 0000000..5362aa3
--- /dev/null
+++ b/src/math/cvt-f32-qs8-neonv8.c
@@ -0,0 +1,36 @@
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/math-stubs.h>
+
+
+void xnn_math_f32_qs8_cvt__neonv8(
+ size_t n,
+ const float* input,
+ int8_t* output,
+ int8_t output_zero_point)
+{
+ assert(n % (8 * sizeof(int8_t)) == 0);
+
+ const int16x8_t voutput_zero_point = vdupq_n_s16((int16_t) output_zero_point);
+ for (; n != 0; n -= 8 * sizeof(int8_t)) {
+ const float32x4_t vx_lo = vld1q_f32(input); input += 4;
+ const float32x4_t vx_hi = vld1q_f32(input); input += 4;
+
+ const int32x4_t vy_lo = vcvtnq_s32_f32(vx_lo);
+ const int32x4_t vy_hi = vcvtnq_s32_f32(vx_hi);
+
+ const int16x8_t vy = vqaddq_s16(vcombine_s16(vqmovn_s32(vy_lo), vqmovn_s32(vy_hi)), voutput_zero_point);
+
+ const int8x8_t vout = vqmovn_s16(vy);
+ vst1_s8(output, vout); output += 8;
+ }
+}
diff --git a/src/math/cvt-f32-qu8-neon.c b/src/math/cvt-f32-qu8-neon.c
new file mode 100644
index 0000000..84e9bdd
--- /dev/null
+++ b/src/math/cvt-f32-qu8-neon.c
@@ -0,0 +1,47 @@
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/math-stubs.h>
+
+
+void xnn_math_f32_qu8_cvt__neon(
+ size_t n,
+ const float* input,
+ uint8_t* output,
+ uint8_t output_zero_point)
+{
+ assert(n % (8 * sizeof(uint8_t)) == 0);
+
+ const int32x4_t vmin = vreinterpretq_s32_f32(vdupq_n_f32(12582912.0f - (float) (int32_t) output_zero_point));
+ const float32x4_t vfmagic = vdupq_n_f32(12582912.0f);
+ const int32x4_t vimagic = vdupq_n_s32(INT32_C(0x4B400000) - (int32_t) output_zero_point);
+ for (; n != 0; n -= 8 * sizeof(uint8_t)) {
+ float32x4_t vx_lo = vld1q_f32(input); input += 4;
+ float32x4_t vx_hi = vld1q_f32(input); input += 4;
+
+ vx_lo = vaddq_f32(vx_lo, vfmagic);
+ vx_hi = vaddq_f32(vx_hi, vfmagic);
+
+ int32x4_t vy_lo = vreinterpretq_s32_f32(vx_lo);
+ int32x4_t vy_hi = vreinterpretq_s32_f32(vx_hi);
+
+ vy_lo = vmaxq_s32(vy_lo, vmin);
+ vy_hi = vmaxq_s32(vy_hi, vmin);
+
+ vy_lo = vsubq_s32(vy_lo, vimagic);
+ vy_hi = vsubq_s32(vy_hi, vimagic);
+
+ const int16x8_t vy = vcombine_s16(vqmovn_s32(vy_lo), vqmovn_s32(vy_hi));
+
+ const uint8x8_t vout = vqmovun_s16(vy);
+ vst1_u8(output, vout); output += 8;
+ }
+}
diff --git a/src/math/cvt-f32-qu8-neonv8.c b/src/math/cvt-f32-qu8-neonv8.c
new file mode 100644
index 0000000..cef8ca7
--- /dev/null
+++ b/src/math/cvt-f32-qu8-neonv8.c
@@ -0,0 +1,36 @@
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/math-stubs.h>
+
+
+void xnn_math_f32_qu8_cvt__neonv8(
+ size_t n,
+ const float* input,
+ uint8_t* output,
+ uint8_t output_zero_point)
+{
+ assert(n % (8 * sizeof(int8_t)) == 0);
+
+ const int16x8_t voutput_zero_point = vdupq_n_s16((int16_t) (uint16_t) output_zero_point);
+ for (; n != 0; n -= 8 * sizeof(int8_t)) {
+ const float32x4_t vx_lo = vld1q_f32(input); input += 4;
+ const float32x4_t vx_hi = vld1q_f32(input); input += 4;
+
+ const int32x4_t vy_lo = vcvtnq_s32_f32(vx_lo);
+ const int32x4_t vy_hi = vcvtnq_s32_f32(vx_hi);
+
+ const int16x8_t vy = vqaddq_s16(vcombine_s16(vqmovn_s32(vy_lo), vqmovn_s32(vy_hi)), voutput_zero_point);
+
+ const uint8x8_t vout = vqmovun_s16(vy);
+ vst1_u8(output, vout); output += 8;
+ }
+}
diff --git a/src/xnnpack/math-stubs.h b/src/xnnpack/math-stubs.h
index 39e21b4..d4d4741 100644
--- a/src/xnnpack/math-stubs.h
+++ b/src/xnnpack/math-stubs.h
@@ -41,6 +41,20 @@
const float* input, \
void* output);
+#define DECLARE_F32_QS8_CVT_MATH_FUNCTION(fn_name) \
+ void fn_name( \
+ size_t n, \
+ const float* input, \
+ int8_t* output, \
+ int8_t output_zero_point);
+
+#define DECLARE_F32_QU8_CVT_MATH_FUNCTION(fn_name) \
+ void fn_name( \
+ size_t n, \
+ const float* input, \
+ uint8_t* output, \
+ uint8_t output_zero_point);
+
#define DECLARE_F32_EXT_UNARY_MATH_FUNCTION(fn_name) \
void fn_name( \
size_t n, \
@@ -68,6 +82,12 @@
DECLARE_F32_F16_CVT_MATH_FUNCTION(xnn_math_f32_f16_cvt__scalar_bitcast)
DECLARE_F32_F16_CVT_MATH_FUNCTION(xnn_math_f32_f16_cvt__scalar_fabsf)
+DECLARE_F32_QS8_CVT_MATH_FUNCTION(xnn_math_f32_qs8_cvt__neon)
+DECLARE_F32_QS8_CVT_MATH_FUNCTION(xnn_math_f32_qs8_cvt__neonv8)
+
+DECLARE_F32_QU8_CVT_MATH_FUNCTION(xnn_math_f32_qu8_cvt__neon)
+DECLARE_F32_QU8_CVT_MATH_FUNCTION(xnn_math_f32_qu8_cvt__neonv8)
+
DECLARE_F32_UNARY_MATH_FUNCTION(xnn_math_f32_roundne__neon_addsub)
DECLARE_F32_UNARY_MATH_FUNCTION(xnn_math_f32_roundne__neonv8)
DECLARE_F32_UNARY_MATH_FUNCTION(xnn_math_f32_roundne__sse_addsub)