Mike Klein | 78d5a3b | 2016-09-30 10:48:01 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2016 Google Inc. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license that can be |
| 5 | * found in the LICENSE file. |
| 6 | */ |
| 7 | |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 8 | // It is not safe to #include any header file here unless it has been vetted for ODR safety: |
| 9 | // all symbols used must be file-scoped static or in an anonymous namespace. This applies |
| 10 | // to _all_ header files: C standard library, C++ standard library, Skia... everything. |
Mike Klein | 78d5a3b | 2016-09-30 10:48:01 -0400 | [diff] [blame] | 11 | |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 12 | #include <immintrin.h> // ODR safe |
| 13 | #include <stdint.h> // ODR safe |
Mike Klein | 464e6a1 | 2017-01-04 11:04:01 -0500 | [diff] [blame] | 14 | |
Mike Klein | 88235bb | 2017-01-26 16:23:26 -0500 | [diff] [blame] | 15 | #if defined(__AVX2__) |
| 16 | |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 17 | namespace hsw { |
Mike Klein | 78d5a3b | 2016-09-30 10:48:01 -0400 | [diff] [blame] | 18 | |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 19 | void convolve_vertically(const int16_t* filter, int filterLen, |
| 20 | uint8_t* const* srcRows, int width, |
| 21 | uint8_t* out, bool hasAlpha) { |
| 22 | // It's simpler to work with the output array in terms of 4-byte pixels. |
| 23 | auto dst = (int*)out; |
| 24 | |
| 25 | // Output up to eight pixels per iteration. |
| 26 | for (int x = 0; x < width; x += 8) { |
Mike Klein | f65d19b | 2017-01-26 14:58:42 -0500 | [diff] [blame] | 27 | // Accumulated result for 4 (non-adjacent) pairs of pixels, |
| 28 | // with each channel in signed 17.14 fixed point. |
| 29 | auto accum04 = _mm256_setzero_si256(), |
| 30 | accum15 = _mm256_setzero_si256(), |
| 31 | accum26 = _mm256_setzero_si256(), |
| 32 | accum37 = _mm256_setzero_si256(); |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 33 | |
| 34 | // Convolve with the filter. (This inner loop is where we spend ~all our time.) |
Mike Klein | 690dd5e | 2017-01-26 15:46:02 -0500 | [diff] [blame] | 35 | // While we can, we consume 2 filter coefficients and 2 rows of 8 pixels each at a time. |
| 36 | auto convolve_16_pixels = [&](__m256i interlaced_coeffs, |
| 37 | __m256i pixels_01234567, __m256i pixels_89ABCDEF) { |
| 38 | // Interlaced R0R8 G0G8 B0B8 A0A8 R1R9 G1G9... 32 8-bit values each. |
| 39 | auto _08194C5D = _mm256_unpacklo_epi8(pixels_01234567, pixels_89ABCDEF), |
| 40 | _2A3B6E7F = _mm256_unpackhi_epi8(pixels_01234567, pixels_89ABCDEF); |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 41 | |
Mike Klein | 690dd5e | 2017-01-26 15:46:02 -0500 | [diff] [blame] | 42 | // Still interlaced R0R8 G0G8... as above, each channel expanded to 16-bit lanes. |
| 43 | auto _084C = _mm256_unpacklo_epi8(_08194C5D, _mm256_setzero_si256()), |
| 44 | _195D = _mm256_unpackhi_epi8(_08194C5D, _mm256_setzero_si256()), |
| 45 | _2A6E = _mm256_unpacklo_epi8(_2A3B6E7F, _mm256_setzero_si256()), |
| 46 | _3B7F = _mm256_unpackhi_epi8(_2A3B6E7F, _mm256_setzero_si256()); |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 47 | |
Mike Klein | 690dd5e | 2017-01-26 15:46:02 -0500 | [diff] [blame] | 48 | // accum0_R += R0*coeff0 + R8*coeff1, etc. |
| 49 | accum04 = _mm256_add_epi32(accum04, _mm256_madd_epi16(_084C, interlaced_coeffs)); |
| 50 | accum15 = _mm256_add_epi32(accum15, _mm256_madd_epi16(_195D, interlaced_coeffs)); |
| 51 | accum26 = _mm256_add_epi32(accum26, _mm256_madd_epi16(_2A6E, interlaced_coeffs)); |
| 52 | accum37 = _mm256_add_epi32(accum37, _mm256_madd_epi16(_3B7F, interlaced_coeffs)); |
| 53 | }; |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 54 | |
Mike Klein | 690dd5e | 2017-01-26 15:46:02 -0500 | [diff] [blame] | 55 | int i = 0; |
| 56 | for (; i < filterLen/2*2; i += 2) { |
| 57 | convolve_16_pixels(_mm256_set1_epi32(*(const int32_t*)(filter+i)), |
| 58 | _mm256_loadu_si256((const __m256i*)(srcRows[i+0] + x*4)), |
| 59 | _mm256_loadu_si256((const __m256i*)(srcRows[i+1] + x*4))); |
| 60 | } |
| 61 | if (i < filterLen) { |
| 62 | convolve_16_pixels(_mm256_set1_epi32(*(const int16_t*)(filter+i)), |
| 63 | _mm256_loadu_si256((const __m256i*)(srcRows[i] + x*4)), |
| 64 | _mm256_setzero_si256()); |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 65 | } |
| 66 | |
Mike Klein | 690dd5e | 2017-01-26 15:46:02 -0500 | [diff] [blame] | 67 | // Trim the fractional parts off the accumulators. |
Mike Klein | f65d19b | 2017-01-26 14:58:42 -0500 | [diff] [blame] | 68 | accum04 = _mm256_srai_epi32(accum04, 14); |
| 69 | accum15 = _mm256_srai_epi32(accum15, 14); |
| 70 | accum26 = _mm256_srai_epi32(accum26, 14); |
| 71 | accum37 = _mm256_srai_epi32(accum37, 14); |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 72 | |
| 73 | // Pack back down to 8-bit channels. |
Mike Klein | f65d19b | 2017-01-26 14:58:42 -0500 | [diff] [blame] | 74 | auto pixels = _mm256_packus_epi16(_mm256_packs_epi32(accum04, accum15), |
| 75 | _mm256_packs_epi32(accum26, accum37)); |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 76 | |
| 77 | if (hasAlpha) { |
| 78 | // Clamp alpha to the max of r,g,b to make sure we stay premultiplied. |
| 79 | __m256i max_rg = _mm256_max_epu8(pixels, _mm256_srli_epi32(pixels, 8)), |
| 80 | max_rgb = _mm256_max_epu8(max_rg, _mm256_srli_epi32(pixels, 16)); |
| 81 | pixels = _mm256_max_epu8(pixels, _mm256_slli_epi32(max_rgb, 24)); |
| 82 | } else { |
| 83 | // Force opaque. |
| 84 | pixels = _mm256_or_si256(pixels, _mm256_set1_epi32(0xff000000)); |
| 85 | } |
| 86 | |
| 87 | // Normal path to store 8 pixels. |
| 88 | if (x + 8 <= width) { |
| 89 | _mm256_storeu_si256((__m256i*)dst, pixels); |
| 90 | dst += 8; |
| 91 | continue; |
| 92 | } |
| 93 | |
| 94 | // Store one pixel at a time on the last iteration. |
| 95 | for (int i = x; i < width; i++) { |
| 96 | *dst++ = _mm_cvtsi128_si32(_mm256_castsi256_si128(pixels)); |
| 97 | pixels = _mm256_permutevar8x32_epi32(pixels, _mm256_setr_epi32(1,2,3,4,5,6,7,0)); |
| 98 | } |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | } |
Mike Klein | e9f78b4 | 2016-11-22 08:57:45 -0500 | [diff] [blame] | 103 | |
Mike Klein | 78d5a3b | 2016-09-30 10:48:01 -0400 | [diff] [blame] | 104 | namespace SkOpts { |
Mike Klein | a174e06 | 2017-01-26 11:41:03 -0500 | [diff] [blame] | 105 | // See SkOpts.h, writing SkConvolutionFilter1D::ConvolutionFixed as the underlying type. |
| 106 | extern void (*convolve_vertically)(const int16_t* filter, int filterLen, |
| 107 | uint8_t* const* srcRows, int width, |
| 108 | uint8_t* out, bool hasAlpha); |
Mike Klein | ec07b0b | 2016-10-19 16:45:16 -0400 | [diff] [blame] | 109 | void Init_hsw() { |
xiangze.zhang | 4adac2e | 2016-12-07 17:54:04 -0800 | [diff] [blame] | 110 | convolve_vertically = hsw::convolve_vertically; |
Mike Klein | ec07b0b | 2016-10-19 16:45:16 -0400 | [diff] [blame] | 111 | } |
Mike Klein | 78d5a3b | 2016-09-30 10:48:01 -0400 | [diff] [blame] | 112 | } |
Mike Klein | 88235bb | 2017-01-26 16:23:26 -0500 | [diff] [blame] | 113 | |
| 114 | #else // defined(__AVX2__) is not true... |
| 115 | |
| 116 | namespace SkOpts { void Init_hsw() {} } |
| 117 | |
| 118 | #endif |