mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2015 Google Inc. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license that can be |
| 5 | * found in the LICENSE file. |
| 6 | */ |
| 7 | |
| 8 | #ifndef SkNx_neon_DEFINED |
| 9 | #define SkNx_neon_DEFINED |
| 10 | |
fmalita | c2e0ac4 | 2015-12-03 09:15:25 -0800 | [diff] [blame] | 11 | #define SKNX_IS_FAST |
| 12 | |
mtklein | 86c6c49 | 2016-02-09 13:46:49 -0800 | [diff] [blame^] | 13 | // ARMv8 has vrndmq_f32 to floor 4 floats. Here we emulate it: |
| 14 | // - round by adding (1<<23) with our sign, then subtracting it; |
| 15 | // - if that rounded value is bigger than our input, subtract 1. |
| 16 | static inline float32x4_t armv7_vrndmq_f32(float32x4_t v) { |
| 17 | auto sign = vandq_u32((uint32x4_t)v, vdupq_n_u32(1<<31)); |
| 18 | auto bias = (float32x4_t)(vorrq_u32((uint32x4_t)vdupq_n_f32(1<<23), sign)); |
| 19 | auto rounded = vsubq_f32(vaddq_f32(v, bias), bias); |
| 20 | auto too_big = vcgtq_f32(rounded, v); |
| 21 | return vsubq_f32(rounded, (float32x4_t)vandq_u32(too_big, (uint32x4_t)vdupq_n_f32(1))); |
| 22 | } |
| 23 | |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 24 | // Well, this is absurd. The shifts require compile-time constant arguments. |
| 25 | |
| 26 | #define SHIFT8(op, v, bits) switch(bits) { \ |
| 27 | case 1: return op(v, 1); case 2: return op(v, 2); case 3: return op(v, 3); \ |
| 28 | case 4: return op(v, 4); case 5: return op(v, 5); case 6: return op(v, 6); \ |
| 29 | case 7: return op(v, 7); \ |
| 30 | } return fVec |
| 31 | |
| 32 | #define SHIFT16(op, v, bits) if (bits < 8) { SHIFT8(op, v, bits); } switch(bits) { \ |
| 33 | case 8: return op(v, 8); case 9: return op(v, 9); \ |
| 34 | case 10: return op(v, 10); case 11: return op(v, 11); case 12: return op(v, 12); \ |
| 35 | case 13: return op(v, 13); case 14: return op(v, 14); case 15: return op(v, 15); \ |
| 36 | } return fVec |
| 37 | |
| 38 | #define SHIFT32(op, v, bits) if (bits < 16) { SHIFT16(op, v, bits); } switch(bits) { \ |
| 39 | case 16: return op(v, 16); case 17: return op(v, 17); case 18: return op(v, 18); \ |
| 40 | case 19: return op(v, 19); case 20: return op(v, 20); case 21: return op(v, 21); \ |
| 41 | case 22: return op(v, 22); case 23: return op(v, 23); case 24: return op(v, 24); \ |
| 42 | case 25: return op(v, 25); case 26: return op(v, 26); case 27: return op(v, 27); \ |
| 43 | case 28: return op(v, 28); case 29: return op(v, 29); case 30: return op(v, 30); \ |
| 44 | case 31: return op(v, 31); } return fVec |
| 45 | |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 46 | template <> |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 47 | class SkNx<2, float> { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 48 | public: |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 49 | SkNx(float32x2_t vec) : fVec(vec) {} |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 50 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 51 | SkNx() {} |
| 52 | SkNx(float val) : fVec(vdup_n_f32(val)) {} |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 53 | static SkNx Load(const void* ptr) { return vld1_f32((const float*)ptr); } |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 54 | SkNx(float a, float b) { fVec = (float32x2_t) { a, b }; } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 55 | |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 56 | void store(void* ptr) const { vst1_f32((float*)ptr, fVec); } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 57 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 58 | SkNx approxInvert() const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 59 | float32x2_t est0 = vrecpe_f32(fVec), |
| 60 | est1 = vmul_f32(vrecps_f32(est0, fVec), est0); |
| 61 | return est1; |
| 62 | } |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 63 | SkNx invert() const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 64 | float32x2_t est1 = this->approxInvert().fVec, |
| 65 | est2 = vmul_f32(vrecps_f32(est1, fVec), est1); |
| 66 | return est2; |
| 67 | } |
| 68 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 69 | SkNx operator + (const SkNx& o) const { return vadd_f32(fVec, o.fVec); } |
| 70 | SkNx operator - (const SkNx& o) const { return vsub_f32(fVec, o.fVec); } |
| 71 | SkNx operator * (const SkNx& o) const { return vmul_f32(fVec, o.fVec); } |
| 72 | SkNx operator / (const SkNx& o) const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 73 | #if defined(SK_CPU_ARM64) |
| 74 | return vdiv_f32(fVec, o.fVec); |
| 75 | #else |
| 76 | return vmul_f32(fVec, o.invert().fVec); |
| 77 | #endif |
| 78 | } |
| 79 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 80 | SkNx operator == (const SkNx& o) const { return vreinterpret_f32_u32(vceq_f32(fVec, o.fVec)); } |
| 81 | SkNx operator < (const SkNx& o) const { return vreinterpret_f32_u32(vclt_f32(fVec, o.fVec)); } |
| 82 | SkNx operator > (const SkNx& o) const { return vreinterpret_f32_u32(vcgt_f32(fVec, o.fVec)); } |
| 83 | SkNx operator <= (const SkNx& o) const { return vreinterpret_f32_u32(vcle_f32(fVec, o.fVec)); } |
| 84 | SkNx operator >= (const SkNx& o) const { return vreinterpret_f32_u32(vcge_f32(fVec, o.fVec)); } |
| 85 | SkNx operator != (const SkNx& o) const { |
mtklein | b5e8611 | 2015-06-24 15:18:39 -0700 | [diff] [blame] | 86 | return vreinterpret_f32_u32(vmvn_u32(vceq_f32(fVec, o.fVec))); |
| 87 | } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 88 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 89 | static SkNx Min(const SkNx& l, const SkNx& r) { return vmin_f32(l.fVec, r.fVec); } |
| 90 | static SkNx Max(const SkNx& l, const SkNx& r) { return vmax_f32(l.fVec, r.fVec); } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 91 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 92 | SkNx rsqrt0() const { return vrsqrte_f32(fVec); } |
| 93 | SkNx rsqrt1() const { |
mtklein | d7c014f | 2015-04-27 14:22:32 -0700 | [diff] [blame] | 94 | float32x2_t est0 = this->rsqrt0().fVec; |
| 95 | return vmul_f32(vrsqrts_f32(fVec, vmul_f32(est0, est0)), est0); |
| 96 | } |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 97 | SkNx rsqrt2() const { |
mtklein | d7c014f | 2015-04-27 14:22:32 -0700 | [diff] [blame] | 98 | float32x2_t est1 = this->rsqrt1().fVec; |
| 99 | return vmul_f32(vrsqrts_f32(fVec, vmul_f32(est1, est1)), est1); |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 100 | } |
| 101 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 102 | SkNx sqrt() const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 103 | #if defined(SK_CPU_ARM64) |
| 104 | return vsqrt_f32(fVec); |
| 105 | #else |
mtklein | d7c014f | 2015-04-27 14:22:32 -0700 | [diff] [blame] | 106 | return *this * this->rsqrt2(); |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 107 | #endif |
| 108 | } |
| 109 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 110 | float operator[](int k) const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 111 | SkASSERT(0 <= k && k < 2); |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 112 | union { float32x2_t v; float fs[2]; } pun = {fVec}; |
| 113 | return pun.fs[k&1]; |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 114 | } |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 115 | template <int k> float kth() const { return (*this)[k]; } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 116 | |
mtklein | b5e8611 | 2015-06-24 15:18:39 -0700 | [diff] [blame] | 117 | bool allTrue() const { |
| 118 | auto v = vreinterpret_u32_f32(fVec); |
| 119 | return vget_lane_u32(v,0) && vget_lane_u32(v,1); |
| 120 | } |
| 121 | bool anyTrue() const { |
| 122 | auto v = vreinterpret_u32_f32(fVec); |
| 123 | return vget_lane_u32(v,0) || vget_lane_u32(v,1); |
| 124 | } |
| 125 | |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 126 | float32x2_t fVec; |
| 127 | }; |
| 128 | |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 129 | template <> |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 130 | class SkNx<4, float> { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 131 | public: |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 132 | SkNx(float32x4_t vec) : fVec(vec) {} |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 133 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 134 | SkNx() {} |
| 135 | SkNx(float val) : fVec(vdupq_n_f32(val)) {} |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 136 | static SkNx Load(const void* ptr) { return vld1q_f32((const float*)ptr); } |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 137 | SkNx(float a, float b, float c, float d) { fVec = (float32x4_t) { a, b, c, d }; } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 138 | |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 139 | void store(void* ptr) const { vst1q_f32((float*)ptr, fVec); } |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 140 | SkNx approxInvert() const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 141 | float32x4_t est0 = vrecpeq_f32(fVec), |
| 142 | est1 = vmulq_f32(vrecpsq_f32(est0, fVec), est0); |
| 143 | return est1; |
| 144 | } |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 145 | SkNx invert() const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 146 | float32x4_t est1 = this->approxInvert().fVec, |
| 147 | est2 = vmulq_f32(vrecpsq_f32(est1, fVec), est1); |
| 148 | return est2; |
| 149 | } |
| 150 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 151 | SkNx operator + (const SkNx& o) const { return vaddq_f32(fVec, o.fVec); } |
| 152 | SkNx operator - (const SkNx& o) const { return vsubq_f32(fVec, o.fVec); } |
| 153 | SkNx operator * (const SkNx& o) const { return vmulq_f32(fVec, o.fVec); } |
| 154 | SkNx operator / (const SkNx& o) const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 155 | #if defined(SK_CPU_ARM64) |
| 156 | return vdivq_f32(fVec, o.fVec); |
| 157 | #else |
| 158 | return vmulq_f32(fVec, o.invert().fVec); |
| 159 | #endif |
| 160 | } |
| 161 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 162 | SkNx operator==(const SkNx& o) const { return vreinterpretq_f32_u32(vceqq_f32(fVec, o.fVec)); } |
| 163 | SkNx operator <(const SkNx& o) const { return vreinterpretq_f32_u32(vcltq_f32(fVec, o.fVec)); } |
| 164 | SkNx operator >(const SkNx& o) const { return vreinterpretq_f32_u32(vcgtq_f32(fVec, o.fVec)); } |
| 165 | SkNx operator<=(const SkNx& o) const { return vreinterpretq_f32_u32(vcleq_f32(fVec, o.fVec)); } |
| 166 | SkNx operator>=(const SkNx& o) const { return vreinterpretq_f32_u32(vcgeq_f32(fVec, o.fVec)); } |
| 167 | SkNx operator!=(const SkNx& o) const { |
mtklein | b5e8611 | 2015-06-24 15:18:39 -0700 | [diff] [blame] | 168 | return vreinterpretq_f32_u32(vmvnq_u32(vceqq_f32(fVec, o.fVec))); |
| 169 | } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 170 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 171 | static SkNx Min(const SkNx& l, const SkNx& r) { return vminq_f32(l.fVec, r.fVec); } |
| 172 | static SkNx Max(const SkNx& l, const SkNx& r) { return vmaxq_f32(l.fVec, r.fVec); } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 173 | |
mtklein | c33065a | 2016-01-15 12:16:40 -0800 | [diff] [blame] | 174 | SkNx abs() const { return vabsq_f32(fVec); } |
mtklein | 86c6c49 | 2016-02-09 13:46:49 -0800 | [diff] [blame^] | 175 | SkNx floor() const { |
| 176 | #if defined(SK_CPU_ARM64) |
| 177 | return vrndmq_f32(fVec); |
| 178 | #else |
| 179 | return armv7_vrndmq_f32(fVec); |
| 180 | #endif |
| 181 | } |
| 182 | |
mtklein | c33065a | 2016-01-15 12:16:40 -0800 | [diff] [blame] | 183 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 184 | SkNx rsqrt0() const { return vrsqrteq_f32(fVec); } |
| 185 | SkNx rsqrt1() const { |
mtklein | d7c014f | 2015-04-27 14:22:32 -0700 | [diff] [blame] | 186 | float32x4_t est0 = this->rsqrt0().fVec; |
| 187 | return vmulq_f32(vrsqrtsq_f32(fVec, vmulq_f32(est0, est0)), est0); |
| 188 | } |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 189 | SkNx rsqrt2() const { |
mtklein | d7c014f | 2015-04-27 14:22:32 -0700 | [diff] [blame] | 190 | float32x4_t est1 = this->rsqrt1().fVec; |
| 191 | return vmulq_f32(vrsqrtsq_f32(fVec, vmulq_f32(est1, est1)), est1); |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 192 | } |
| 193 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 194 | SkNx sqrt() const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 195 | #if defined(SK_CPU_ARM64) |
| 196 | return vsqrtq_f32(fVec); |
| 197 | #else |
mtklein | d7c014f | 2015-04-27 14:22:32 -0700 | [diff] [blame] | 198 | return *this * this->rsqrt2(); |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 199 | #endif |
| 200 | } |
| 201 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 202 | float operator[](int k) const { |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 203 | SkASSERT(0 <= k && k < 4); |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 204 | union { float32x4_t v; float fs[4]; } pun = {fVec}; |
| 205 | return pun.fs[k&3]; |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 206 | } |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 207 | template <int k> float kth() const { return (*this)[k]; } |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 208 | |
mtklein | b5e8611 | 2015-06-24 15:18:39 -0700 | [diff] [blame] | 209 | bool allTrue() const { |
| 210 | auto v = vreinterpretq_u32_f32(fVec); |
| 211 | return vgetq_lane_u32(v,0) && vgetq_lane_u32(v,1) |
| 212 | && vgetq_lane_u32(v,2) && vgetq_lane_u32(v,3); |
| 213 | } |
| 214 | bool anyTrue() const { |
| 215 | auto v = vreinterpretq_u32_f32(fVec); |
| 216 | return vgetq_lane_u32(v,0) || vgetq_lane_u32(v,1) |
| 217 | || vgetq_lane_u32(v,2) || vgetq_lane_u32(v,3); |
| 218 | } |
| 219 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 220 | SkNx thenElse(const SkNx& t, const SkNx& e) const { |
mtklein | cf4e567 | 2015-07-27 06:12:05 -0700 | [diff] [blame] | 221 | return vbslq_f32(vreinterpretq_u32_f32(fVec), t.fVec, e.fVec); |
mtklein | 2aab22a | 2015-06-26 10:46:31 -0700 | [diff] [blame] | 222 | } |
| 223 | |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 224 | float32x4_t fVec; |
| 225 | }; |
| 226 | |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 227 | // It's possible that for our current use cases, representing this as |
| 228 | // half a uint16x8_t might be better than representing it as a uint16x4_t. |
| 229 | // It'd make conversion to Sk4b one step simpler. |
| 230 | template <> |
| 231 | class SkNx<4, uint16_t> { |
| 232 | public: |
| 233 | SkNx(const uint16x4_t& vec) : fVec(vec) {} |
| 234 | |
| 235 | SkNx() {} |
| 236 | SkNx(uint16_t val) : fVec(vdup_n_u16(val)) {} |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 237 | static SkNx Load(const void* ptr) { return vld1_u16((const uint16_t*)ptr); } |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 238 | |
| 239 | SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { |
| 240 | fVec = (uint16x4_t) { a,b,c,d }; |
| 241 | } |
| 242 | |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 243 | void store(void* ptr) const { vst1_u16((uint16_t*)ptr, fVec); } |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 244 | |
| 245 | SkNx operator + (const SkNx& o) const { return vadd_u16(fVec, o.fVec); } |
| 246 | SkNx operator - (const SkNx& o) const { return vsub_u16(fVec, o.fVec); } |
| 247 | SkNx operator * (const SkNx& o) const { return vmul_u16(fVec, o.fVec); } |
| 248 | |
| 249 | SkNx operator << (int bits) const { SHIFT16(vshl_n_u16, fVec, bits); } |
| 250 | SkNx operator >> (int bits) const { SHIFT16(vshr_n_u16, fVec, bits); } |
| 251 | |
| 252 | static SkNx Min(const SkNx& a, const SkNx& b) { return vmin_u16(a.fVec, b.fVec); } |
| 253 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 254 | uint16_t operator[](int k) const { |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 255 | SkASSERT(0 <= k && k < 4); |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 256 | union { uint16x4_t v; uint16_t us[4]; } pun = {fVec}; |
| 257 | return pun.us[k&3]; |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 258 | } |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 259 | template <int k> uint16_t kth() const { return (*this)[k]; } |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 260 | |
| 261 | SkNx thenElse(const SkNx& t, const SkNx& e) const { |
| 262 | return vbsl_u16(fVec, t.fVec, e.fVec); |
| 263 | } |
| 264 | |
| 265 | uint16x4_t fVec; |
| 266 | }; |
| 267 | |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 268 | template <> |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 269 | class SkNx<8, uint16_t> { |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 270 | public: |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 271 | SkNx(const uint16x8_t& vec) : fVec(vec) {} |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 272 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 273 | SkNx() {} |
| 274 | SkNx(uint16_t val) : fVec(vdupq_n_u16(val)) {} |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 275 | static SkNx Load(const void* ptr) { return vld1q_u16((const uint16_t*)ptr); } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 276 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 277 | SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d, |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 278 | uint16_t e, uint16_t f, uint16_t g, uint16_t h) { |
| 279 | fVec = (uint16x8_t) { a,b,c,d, e,f,g,h }; |
| 280 | } |
| 281 | |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 282 | void store(void* ptr) const { vst1q_u16((uint16_t*)ptr, fVec); } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 283 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 284 | SkNx operator + (const SkNx& o) const { return vaddq_u16(fVec, o.fVec); } |
| 285 | SkNx operator - (const SkNx& o) const { return vsubq_u16(fVec, o.fVec); } |
| 286 | SkNx operator * (const SkNx& o) const { return vmulq_u16(fVec, o.fVec); } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 287 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 288 | SkNx operator << (int bits) const { SHIFT16(vshlq_n_u16, fVec, bits); } |
| 289 | SkNx operator >> (int bits) const { SHIFT16(vshrq_n_u16, fVec, bits); } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 290 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 291 | static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u16(a.fVec, b.fVec); } |
mtklein | 27e517a | 2015-05-14 17:53:04 -0700 | [diff] [blame] | 292 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 293 | uint16_t operator[](int k) const { |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 294 | SkASSERT(0 <= k && k < 8); |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 295 | union { uint16x8_t v; uint16_t us[8]; } pun = {fVec}; |
| 296 | return pun.us[k&7]; |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 297 | } |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 298 | template <int k> uint16_t kth() const { return (*this)[k]; } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 299 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 300 | SkNx thenElse(const SkNx& t, const SkNx& e) const { |
mtklein | cf4e567 | 2015-07-27 06:12:05 -0700 | [diff] [blame] | 301 | return vbslq_u16(fVec, t.fVec, e.fVec); |
mtklein | 4be181e | 2015-07-14 10:54:19 -0700 | [diff] [blame] | 302 | } |
| 303 | |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 304 | uint16x8_t fVec; |
| 305 | }; |
| 306 | |
| 307 | template <> |
mtklein | 6f37b4a | 2015-12-14 11:25:18 -0800 | [diff] [blame] | 308 | class SkNx<4, uint8_t> { |
| 309 | public: |
| 310 | SkNx(const uint8x8_t& vec) : fVec(vec) {} |
| 311 | |
| 312 | SkNx() {} |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 313 | static SkNx Load(const void* ptr) { |
| 314 | return (uint8x8_t)vld1_dup_u32((const uint32_t*)ptr); |
mtklein | 6f37b4a | 2015-12-14 11:25:18 -0800 | [diff] [blame] | 315 | } |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 316 | void store(void* ptr) const { |
| 317 | return vst1_lane_u32((uint32_t*)ptr, (uint32x2_t)fVec, 0); |
mtklein | 6f37b4a | 2015-12-14 11:25:18 -0800 | [diff] [blame] | 318 | } |
| 319 | |
| 320 | // TODO as needed |
| 321 | |
| 322 | uint8x8_t fVec; |
| 323 | }; |
| 324 | |
| 325 | template <> |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 326 | class SkNx<16, uint8_t> { |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 327 | public: |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 328 | SkNx(const uint8x16_t& vec) : fVec(vec) {} |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 329 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 330 | SkNx() {} |
| 331 | SkNx(uint8_t val) : fVec(vdupq_n_u8(val)) {} |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 332 | static SkNx Load(const void* ptr) { return vld1q_u8((const uint8_t*)ptr); } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 333 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 334 | SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d, |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 335 | uint8_t e, uint8_t f, uint8_t g, uint8_t h, |
| 336 | uint8_t i, uint8_t j, uint8_t k, uint8_t l, |
| 337 | uint8_t m, uint8_t n, uint8_t o, uint8_t p) { |
| 338 | fVec = (uint8x16_t) { a,b,c,d, e,f,g,h, i,j,k,l, m,n,o,p }; |
| 339 | } |
| 340 | |
mtklein | 507ef6d | 2016-01-31 08:02:47 -0800 | [diff] [blame] | 341 | void store(void* ptr) const { vst1q_u8((uint8_t*)ptr, fVec); } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 342 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 343 | SkNx saturatedAdd(const SkNx& o) const { return vqaddq_u8(fVec, o.fVec); } |
mtklein | 04d24a3 | 2015-05-13 08:02:14 -0700 | [diff] [blame] | 344 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 345 | SkNx operator + (const SkNx& o) const { return vaddq_u8(fVec, o.fVec); } |
| 346 | SkNx operator - (const SkNx& o) const { return vsubq_u8(fVec, o.fVec); } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 347 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 348 | static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u8(a.fVec, b.fVec); } |
| 349 | SkNx operator < (const SkNx& o) const { return vcltq_u8(fVec, o.fVec); } |
mtklein | 27e517a | 2015-05-14 17:53:04 -0700 | [diff] [blame] | 350 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 351 | uint8_t operator[](int k) const { |
| 352 | SkASSERT(0 <= k && k < 16); |
| 353 | union { uint8x16_t v; uint8_t us[16]; } pun = {fVec}; |
| 354 | return pun.us[k&15]; |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 355 | } |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 356 | template <int k> uint8_t kth() const { return (*this)[k]; } |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 357 | |
mtklein | 6c221b4 | 2015-11-20 13:53:19 -0800 | [diff] [blame] | 358 | SkNx thenElse(const SkNx& t, const SkNx& e) const { |
mtklein | cf4e567 | 2015-07-27 06:12:05 -0700 | [diff] [blame] | 359 | return vbslq_u8(fVec, t.fVec, e.fVec); |
mtklein | b5e8611 | 2015-06-24 15:18:39 -0700 | [diff] [blame] | 360 | } |
| 361 | |
mtklein | d2ffd36 | 2015-05-12 06:11:21 -0700 | [diff] [blame] | 362 | uint8x16_t fVec; |
| 363 | }; |
| 364 | |
| 365 | #undef SHIFT32 |
| 366 | #undef SHIFT16 |
| 367 | #undef SHIFT8 |
| 368 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 369 | template<> inline Sk4b SkNx_cast<uint8_t, float>(const Sk4f& src) { |
mtklein | 6f37b4a | 2015-12-14 11:25:18 -0800 | [diff] [blame] | 370 | uint32x4_t _32 = vcvtq_u32_f32(src.fVec); |
| 371 | uint16x4_t _16 = vqmovn_u32(_32); |
| 372 | return vqmovn_u16(vcombine_u16(_16, _16)); |
| 373 | } |
| 374 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 375 | template<> inline Sk4f SkNx_cast<float, uint8_t>(const Sk4b& src) { |
mtklein | 6f37b4a | 2015-12-14 11:25:18 -0800 | [diff] [blame] | 376 | uint16x8_t _16 = vmovl_u8 (src.fVec) ; |
| 377 | uint32x4_t _32 = vmovl_u16(vget_low_u16(_16)); |
| 378 | return vcvtq_f32_u32(_32); |
| 379 | } |
| 380 | |
| 381 | static inline void Sk4f_ToBytes(uint8_t bytes[16], |
| 382 | const Sk4f& a, const Sk4f& b, const Sk4f& c, const Sk4f& d) { |
| 383 | vst1q_u8(bytes, vuzpq_u8(vuzpq_u8((uint8x16_t)vcvtq_u32_f32(a.fVec), |
| 384 | (uint8x16_t)vcvtq_u32_f32(b.fVec)).val[0], |
| 385 | vuzpq_u8((uint8x16_t)vcvtq_u32_f32(c.fVec), |
| 386 | (uint8x16_t)vcvtq_u32_f32(d.fVec)).val[0]).val[0]); |
| 387 | } |
| 388 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 389 | template<> inline Sk4h SkNx_cast<uint16_t, uint8_t>(const Sk4b& src) { |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 390 | return vget_low_u16(vmovl_u8(src.fVec)); |
| 391 | } |
| 392 | |
mtklein | e4c0bee | 2016-02-09 10:35:27 -0800 | [diff] [blame] | 393 | template<> inline Sk4b SkNx_cast<uint8_t, uint16_t>(const Sk4h& src) { |
mtklein | 550e9b0 | 2016-01-20 11:55:51 -0800 | [diff] [blame] | 394 | return vmovn_u16(vcombine_u16(src.fVec, src.fVec)); |
| 395 | } |
| 396 | |
mtklein | c9adb05 | 2015-03-30 10:50:27 -0700 | [diff] [blame] | 397 | #endif//SkNx_neon_DEFINED |