blob: 843dcb59c6ec7cdad6965b046d3b59f87d68abbb [file] [log] [blame]
Mike Klein78d5a3b2016-09-30 10:48:01 -04001/*
2 * Copyright 2016 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
Mike Kleina174e062017-01-26 11:41:03 -05008// It is not safe to #include any header file here unless it has been vetted for ODR safety:
9// all symbols used must be file-scoped static or in an anonymous namespace. This applies
10// to _all_ header files: C standard library, C++ standard library, Skia... everything.
Mike Klein78d5a3b2016-09-30 10:48:01 -040011
Mike Kleina174e062017-01-26 11:41:03 -050012#include <immintrin.h> // ODR safe
13#include <stdint.h> // ODR safe
Mike Klein464e6a12017-01-04 11:04:01 -050014
Mike Kleina174e062017-01-26 11:41:03 -050015namespace hsw {
Mike Klein78d5a3b2016-09-30 10:48:01 -040016
Mike Kleina174e062017-01-26 11:41:03 -050017 void convolve_vertically(const int16_t* filter, int filterLen,
18 uint8_t* const* srcRows, int width,
19 uint8_t* out, bool hasAlpha) {
20 // It's simpler to work with the output array in terms of 4-byte pixels.
21 auto dst = (int*)out;
22
23 // Output up to eight pixels per iteration.
24 for (int x = 0; x < width; x += 8) {
Mike Kleinf65d19b2017-01-26 14:58:42 -050025 // Accumulated result for 4 (non-adjacent) pairs of pixels,
26 // with each channel in signed 17.14 fixed point.
27 auto accum04 = _mm256_setzero_si256(),
28 accum15 = _mm256_setzero_si256(),
29 accum26 = _mm256_setzero_si256(),
30 accum37 = _mm256_setzero_si256();
Mike Kleina174e062017-01-26 11:41:03 -050031
32 // Convolve with the filter. (This inner loop is where we spend ~all our time.)
33 for (int i = 0; i < filterLen; i++) {
34 auto coeffs = _mm256_set1_epi16(filter[i]);
35 auto pixels = _mm256_loadu_si256((const __m256i*)(srcRows[i] + x*4));
36
Mike Kleinf65d19b2017-01-26 14:58:42 -050037 auto pixels_0145 = _mm256_unpacklo_epi8(pixels, _mm256_setzero_si256()),
38 pixels_2367 = _mm256_unpackhi_epi8(pixels, _mm256_setzero_si256());
Mike Kleina174e062017-01-26 11:41:03 -050039
Mike Kleinf65d19b2017-01-26 14:58:42 -050040 auto lo_0145 = _mm256_mullo_epi16(pixels_0145, coeffs),
41 hi_0145 = _mm256_mulhi_epi16(pixels_0145, coeffs),
42 lo_2367 = _mm256_mullo_epi16(pixels_2367, coeffs),
43 hi_2367 = _mm256_mulhi_epi16(pixels_2367, coeffs);
Mike Kleina174e062017-01-26 11:41:03 -050044
Mike Kleinf65d19b2017-01-26 14:58:42 -050045 accum04 = _mm256_add_epi32(accum04, _mm256_unpacklo_epi16(lo_0145, hi_0145));
46 accum15 = _mm256_add_epi32(accum15, _mm256_unpackhi_epi16(lo_0145, hi_0145));
47 accum26 = _mm256_add_epi32(accum26, _mm256_unpacklo_epi16(lo_2367, hi_2367));
48 accum37 = _mm256_add_epi32(accum37, _mm256_unpackhi_epi16(lo_2367, hi_2367));
Mike Kleina174e062017-01-26 11:41:03 -050049 }
50
51 // Trim the fractional parts.
Mike Kleinf65d19b2017-01-26 14:58:42 -050052 accum04 = _mm256_srai_epi32(accum04, 14);
53 accum15 = _mm256_srai_epi32(accum15, 14);
54 accum26 = _mm256_srai_epi32(accum26, 14);
55 accum37 = _mm256_srai_epi32(accum37, 14);
Mike Kleina174e062017-01-26 11:41:03 -050056
57 // Pack back down to 8-bit channels.
Mike Kleinf65d19b2017-01-26 14:58:42 -050058 auto pixels = _mm256_packus_epi16(_mm256_packs_epi32(accum04, accum15),
59 _mm256_packs_epi32(accum26, accum37));
Mike Kleina174e062017-01-26 11:41:03 -050060
61 if (hasAlpha) {
62 // Clamp alpha to the max of r,g,b to make sure we stay premultiplied.
63 __m256i max_rg = _mm256_max_epu8(pixels, _mm256_srli_epi32(pixels, 8)),
64 max_rgb = _mm256_max_epu8(max_rg, _mm256_srli_epi32(pixels, 16));
65 pixels = _mm256_max_epu8(pixels, _mm256_slli_epi32(max_rgb, 24));
66 } else {
67 // Force opaque.
68 pixels = _mm256_or_si256(pixels, _mm256_set1_epi32(0xff000000));
69 }
70
71 // Normal path to store 8 pixels.
72 if (x + 8 <= width) {
73 _mm256_storeu_si256((__m256i*)dst, pixels);
74 dst += 8;
75 continue;
76 }
77
78 // Store one pixel at a time on the last iteration.
79 for (int i = x; i < width; i++) {
80 *dst++ = _mm_cvtsi128_si32(_mm256_castsi256_si128(pixels));
81 pixels = _mm256_permutevar8x32_epi32(pixels, _mm256_setr_epi32(1,2,3,4,5,6,7,0));
82 }
83 }
84 }
85
86}
Mike Kleine9f78b42016-11-22 08:57:45 -050087
Mike Klein78d5a3b2016-09-30 10:48:01 -040088namespace SkOpts {
Mike Kleina174e062017-01-26 11:41:03 -050089 // See SkOpts.h, writing SkConvolutionFilter1D::ConvolutionFixed as the underlying type.
90 extern void (*convolve_vertically)(const int16_t* filter, int filterLen,
91 uint8_t* const* srcRows, int width,
92 uint8_t* out, bool hasAlpha);
Mike Kleinec07b0b2016-10-19 16:45:16 -040093 void Init_hsw() {
xiangze.zhang4adac2e2016-12-07 17:54:04 -080094 convolve_vertically = hsw::convolve_vertically;
Mike Kleinec07b0b2016-10-19 16:45:16 -040095 }
Mike Klein78d5a3b2016-09-30 10:48:01 -040096}