Further optimize NEON & SSE2 conversion-based rounding stubs
PiperOrigin-RevId: 311283485
diff --git a/src/math/roundne-sse2-cvt.c b/src/math/roundne-sse2-cvt.c
index 2c30600..530fea9 100644
--- a/src/math/roundne-sse2-cvt.c
+++ b/src/math/roundne-sse2-cvt.c
@@ -18,30 +18,27 @@
{
assert(n % (4 * sizeof(float)) == 0);
- // This magic number with a bit representation 0x80000000 serves two purposes:
- // 1. Extract the sign of a floating-point number.
+ // This magic number serves two purposes:
+ // 1. Set the bit corresponding to the sign of a floating-point number in a bitmask.
// 2. Check if the input to CVTPS2DQ (_mm_cvtps_epi32) is out-of-range, which results in 0x80000000 output.
- const __m128 vmagic = _mm_set1_ps(-0.0f);
+ const __m128i vmagic = _mm_set1_epi32(0x80000000);
for (; n != 0; n -= 4 * sizeof(float)) {
const __m128 vx = _mm_load_ps(input);
input += 4;
- // Extract the sign of the input.
- // We need the sign to preserve negative zero value, which would otherwise get lost in FP->INT->FP conversion.
- const __m128 vsignx = _mm_and_ps(vx, vmagic);
// Convert floating-point value x to integer, with default rounding (to nearest-even).
// If x is beyond [-2**31, 2**31-1] range or x is NaN, the result is -2**31 (0x80000000).
const __m128i vintx = _mm_cvtps_epi32(vx);
- // Compute bitmask for out-of-range conversion input.
- // The bitmask is set to all ones when x is out-of-range for CVTPS2DQ, and also when x == -2**31. The latter case
- // is ok, because this x is already an integer, and can be passed to output as is.
- const __m128 vrndmask = _mm_castsi128_ps(_mm_cmpeq_epi32(vintx, _mm_castps_si128(vmagic)));
+ // Compute bitmask for the bits we want to copy from the rounded x. Other bits will be copied from x.
+ // If x is out-of-range for CVTPS2DQ, we want all bits from x.
+ // If x is in-range for CVTPS2DQ, we want all but the sign bit from the rounded x and the sign bit from x.
+ const __m128 vrndmask = _mm_castsi128_ps(_mm_or_si128(vmagic, _mm_cmpeq_epi32(vintx, vmagic)));
// Convert integer back to floating-point.
// We binary OR the result with the sign of x to restore the sign of negative zero.
- const __m128 vrndx = _mm_or_ps(_mm_cvtepi32_ps(vintx), vsignx);
+ const __m128 vrndx = _mm_cvtepi32_ps(vintx);
// Combine x rounded via conversion to integer and the initial x value.
// For -2**31 < x < 2**31, the result is x rounded via conversion to integer.
diff --git a/src/math/roundz-neon-cvt.c b/src/math/roundz-neon-cvt.c
index 99a7a7c..621b296 100644
--- a/src/math/roundz-neon-cvt.c
+++ b/src/math/roundz-neon-cvt.c
@@ -5,6 +5,7 @@
#include <assert.h>
#include <stddef.h>
+#include <stdint.h>
#include <arm_neon.h>
@@ -30,19 +31,17 @@
// Convert floating-point value x to integer, with rounding towards zero, and then back to floating-point.
// Note: the result is valid only for abs(x) < 2**31, but we further restrict its use to 2**23.
const float32x4_t vrndx = vcvtq_f32_s32(vcvtq_s32_f32(vx));
- // Extract the sign of the input.
- // We need the sign to preserve negative zero value, which would otherwise get lost in FP->INT->FP conversion.
- const uint32x4_t vsignx = vandq_u32(vreinterpretq_u32_f32(vrndx), vsign_mask);
- // Compute bitmask for non-integral input.
- // The bitmask is set to all ones when x is potentially non-integral, and we round it using FP->INT->FP conversion.
- const uint32x4_t vrndmask = vcaltq_f32(vx, vintegral_threshold);
+ // Compute bitmask for the bits we want to copy from the rounded x. Other bits will be copied from x.
+ // If abs(x) is below the integral threshold, use all but the sign bit from the rounded x and the sign bit from x.
+ // If x is guaranteed integral or NaN, use all bits from x.
+ const uint32x4_t vrndmask = vbicq_u32(vcaltq_f32(vx, vintegral_threshold), vsign_mask);
// Combine x rounded towardz zero via FP->INT->FP conversion and the input x value.
// For 0.0 <= x < 2**23, the result is x rounded via FP->INT->FP conversion.
// For -2**23 < x <= -0.0, the result is abs(x) rounded via FP->INT->FP conversion with the sign of x.
// For abs(x) >= 2**23 or NaN inputs, the result is x itself.
- const float32x4_t vy = vbslq_f32(vbicq_u32(vrndmask, vsignx), vrndx, vx);
+ const float32x4_t vy = vbslq_f32(vrndmask, vrndx, vx);
vst1q_f32(output, vy); output += 4;
}
diff --git a/src/math/roundz-sse2-cvt.c b/src/math/roundz-sse2-cvt.c
index a4a3bde..d731c98 100644
--- a/src/math/roundz-sse2-cvt.c
+++ b/src/math/roundz-sse2-cvt.c
@@ -18,30 +18,27 @@
{
assert(n % (4 * sizeof(float)) == 0);
- // This magic number with a bit representation 0x80000000 serves two purposes:
- // 1. Extract the sign of a floating-point number.
- // 2. Check if the input to CVTTPS2DQ (_mm_cvttps_epi32) is out-of-range, which results in 0x80000000 output.
- const __m128 vmagic = _mm_set1_ps(-0.0f);
+ // This magic number serves two purposes:
+ // 1. Set the bit corresponding to the sign of a floating-point number in a bitmask.
+ // 2. Check if the input to CVTTPS2DQ (_mm_cvtps_epi32) is out-of-range, which results in 0x80000000 output.
+ const __m128i vmagic = _mm_set1_epi32(0x80000000);
for (; n != 0; n -= 4 * sizeof(float)) {
const __m128 vx = _mm_load_ps(input);
input += 4;
- // Extract the sign of the input.
- // We need the sign to preserve negative zero value, which would otherwise get lost in FP->INT->FP conversion.
- const __m128 vsignx = _mm_and_ps(vx, vmagic);
- // Convert floating-point value x to integer, with rounding towards zero.
+ // Convert floating-point value x to integer, with default rounding (to nearest-even).
// If x is beyond [-2**31, 2**31-1] range or x is NaN, the result is -2**31 (0x80000000).
const __m128i vintx = _mm_cvttps_epi32(vx);
- // Compute bitmask for out-of-range conversion input.
- // The bitmask is set to all ones when x is out-of-range for CVTTPS2DQ, and also when x == -2**31. The latter case
- // is ok, because this x is already an integer, and can be passed to output as is.
- const __m128 vrndmask = _mm_castsi128_ps(_mm_cmpeq_epi32(vintx, _mm_castps_si128(vmagic)));
+ // Compute bitmask for the bits we want to copy from the rounded x. Other bits will be copied from x.
+ // If x is out-of-range for CVTTPS2DQ, we want all bits from x.
+ // If x is in-range for CVTTPS2DQ, we want all but the sign bit from the rounded x and the sign bit from x.
+ const __m128 vrndmask = _mm_castsi128_ps(_mm_or_si128(vmagic, _mm_cmpeq_epi32(vintx, vmagic)));
// Convert integer back to floating-point.
// We binary OR the result with the sign of x to restore the sign of negative zero.
- const __m128 vrndx = _mm_or_ps(_mm_cvtepi32_ps(vintx), vsignx);
+ const __m128 vrndx = _mm_cvtepi32_ps(vintx);
// Combine x rounded via conversion to integer and the initial x value.
// For -2**31 < x < 2**31, the result is x rounded via conversion to integer.