source/rotate_neon.cc

/*
 *  Copyright (c) 2011 The LibYuv project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "libyuv/basic_types.h"

namespace libyuv {

#if defined(__ARM_NEON__) && !defined(COVERAGE_ENABLED)

void ReverseLine_NEON(const uint8* src, uint8* dst, int width) {
  asm volatile(
    // compute where to start writing destination
    "add         %1, %2\n"

    // work on segments that are multiples of 16
    "lsrs        r3, %2, #4\n"

    // the output is written in two block.  8 bytes followed
    // by another 8.  reading is done sequentially, from left to
    // right.  writing is done from right to left in block sizes
    // %1, the destination pointer is incremented after writing
    // the first of the two blocks.  need to subtract that 8 off
    // along with 16 to get the next location.
    "mov         r3, #-24\n"

    "beq         2f\n"

    // back of destination by the size of the register that is
    // going to be reversed
    "sub         %1, #16\n"

    // the loop needs to run on blocks of 16.  what will be left
    // over is either a negative number, the residuals that need
    // to be done, or 0.  if this isn't subtracted off here the
    // loop will run one extra time.
    "sub         %2, #16\n"

    "1:\n"
      "vld1.8      {q0}, [%0]!\n"                 // src += 16

        // reverse the bytes in the 64 bit segments.  unable to reverse
        // the bytes in the entire 128 bits in one go.
      "vrev64.8    q0, q0\n"

        // because of the inability to reverse the entire 128 bits
        // reverse the writing out of the two 64 bit segments.
      "vst1.8      {d1}, [%1]!\n"
      "vst1.8      {d0}, [%1], r3\n"              // dst -= 16

      "subs        %2, #16\n"
    "bge         1b\n"

    // add 16 back to the counter.  if the result is 0 there is no
    // residuals so jump past
    "adds        %2, #16\n"
    "beq         5f\n"

    "add         %1, #16\n"

    "2:\n"

    "mov         r3, #-3\n"

    "sub         %1, #2\n"
    "subs        %2, #2\n"
    // check for 16*n+1 scenarios where segments_of_2 should not
    // be run, but there is something left over.
    "blt         4f\n"

// do this in neon registers as per
// http://blogs.arm.com/software-enablement/196-coding-for-neon-part-2-dealing-with-leftovers/
    "3:\n"
    "vld2.8      {d0[0], d1[0]}, [%0]!\n"       // src += 2

    "vst1.8      {d1[0]}, [%1]!\n"
    "vst1.8      {d0[0]}, [%1], r3\n"           // dst -= 2

    "subs        %2, #2\n"
    "bge         3b\n"

    "adds        %2, #2\n"
    "beq         5f\n"

    "4:\n"
    "add         %1, #1\n"
    "vld1.8      {d0[0]}, [%0]\n"
    "vst1.8      {d0[0]}, [%1]\n"

    "5:\n"
    : "+r"(src),              // %0
      "+r"(dst),              // %1
      "+r"(width)             // %2
    :
    : "memory", "cc", "r3", "q0"
  );
}

static const uint8 vtbl_4x4_transpose[16] __attribute__((vector_size(16))) =
  { 0,  4,  8, 12,  1,  5,  9, 13,  2,  6, 10, 14,  3,  7, 11, 15 };

void TransposeWx8_NEON(const uint8* src, int src_stride,
                       uint8* dst, int dst_stride,
                       int width) {
  asm volatile(
    // loops are on blocks of 8.  loop will stop when
    // counter gets to or below 0.  starting the counter
    // at w-8 allow for this
    "sub         %4, #8\n"

    // handle 8x8 blocks.  this should be the majority of the plane
    "1:\n"
      "mov         r9, %0\n"

      "vld1.8      {d0}, [r9], %1\n"
      "vld1.8      {d1}, [r9], %1\n"
      "vld1.8      {d2}, [r9], %1\n"
      "vld1.8      {d3}, [r9], %1\n"
      "vld1.8      {d4}, [r9], %1\n"
      "vld1.8      {d5}, [r9], %1\n"
      "vld1.8      {d6}, [r9], %1\n"
      "vld1.8      {d7}, [r9]\n"

      "vtrn.8      d1, d0\n"
      "vtrn.8      d3, d2\n"
      "vtrn.8      d5, d4\n"
      "vtrn.8      d7, d6\n"

      "vtrn.16     d1, d3\n"
      "vtrn.16     d0, d2\n"
      "vtrn.16     d5, d7\n"
      "vtrn.16     d4, d6\n"

      "vtrn.32     d1, d5\n"
      "vtrn.32     d0, d4\n"
      "vtrn.32     d3, d7\n"
      "vtrn.32     d2, d6\n"

      "vrev16.8    q0, q0\n"
      "vrev16.8    q1, q1\n"
      "vrev16.8    q2, q2\n"
      "vrev16.8    q3, q3\n"

      "mov         r9, %2\n"

      "vst1.8      {d1}, [r9], %3\n"
      "vst1.8      {d0}, [r9], %3\n"
      "vst1.8      {d3}, [r9], %3\n"
      "vst1.8      {d2}, [r9], %3\n"
      "vst1.8      {d5}, [r9], %3\n"
      "vst1.8      {d4}, [r9], %3\n"
      "vst1.8      {d7}, [r9], %3\n"
      "vst1.8      {d6}, [r9]\n"

      "add         %0, #8\n"              // src += 8
      "add         %2, %3, lsl #3\n"      // dst += 8 * dst_stride
      "subs        %4,  #8\n"             // w   -= 8
      "bge         1b\n"

    // add 8 back to counter.  if the result is 0 there are
    // no residuals.
    "adds        %4, #8\n"
    "beq         4f\n"

    // some residual, so between 1 and 7 lines left to transpose
    "cmp         %4, #2\n"
    "blt         3f\n"

    "cmp         %4, #4\n"
    "blt         2f\n"

    // 4x8 block
    "mov         r9, %0\n"
    "vld1.32     {d0[0]}, [r9], %1\n"
    "vld1.32     {d0[1]}, [r9], %1\n"
    "vld1.32     {d1[0]}, [r9], %1\n"
    "vld1.32     {d1[1]}, [r9], %1\n"
    "vld1.32     {d2[0]}, [r9], %1\n"
    "vld1.32     {d2[1]}, [r9], %1\n"
    "vld1.32     {d3[0]}, [r9], %1\n"
    "vld1.32     {d3[1]}, [r9]\n"

    "mov         r9, %2\n"

    "vld1.8      {q3}, [%5]\n"

    "vtbl.8      d4, {d0, d1}, d6\n"
    "vtbl.8      d5, {d0, d1}, d7\n"
    "vtbl.8      d0, {d2, d3}, d6\n"
    "vtbl.8      d1, {d2, d3}, d7\n"

    // TODO: rework shuffle above to write
    //       out with 4 instead of 8 writes
    "vst1.32     {d4[0]}, [r9], %3\n"
    "vst1.32     {d4[1]}, [r9], %3\n"
    "vst1.32     {d5[0]}, [r9], %3\n"
    "vst1.32     {d5[1]}, [r9]\n"

    "add         r9, %2, #4\n"
    "vst1.32     {d0[0]}, [r9], %3\n"
    "vst1.32     {d0[1]}, [r9], %3\n"
    "vst1.32     {d1[0]}, [r9], %3\n"
    "vst1.32     {d1[1]}, [r9]\n"

    "add         %0, #4\n"              // src += 4
    "add         %2, %3, lsl #2\n"      // dst += 4 * dst_stride
    "subs        %4,  #4\n"             // w   -= 4
    "beq         4f\n"

    // some residual, check to see if it includes a 2x8 block,
    // or less
    "cmp         %4, #2\n"
    "blt         3f\n"

    // 2x8 block
    "2:\n"
    "mov         r9, %0\n"
    "vld1.16     {d0[0]}, [r9], %1\n"
    "vld1.16     {d1[0]}, [r9], %1\n"
    "vld1.16     {d0[1]}, [r9], %1\n"
    "vld1.16     {d1[1]}, [r9], %1\n"
    "vld1.16     {d0[2]}, [r9], %1\n"
    "vld1.16     {d1[2]}, [r9], %1\n"
    "vld1.16     {d0[3]}, [r9], %1\n"
    "vld1.16     {d1[3]}, [r9]\n"

    "vtrn.8      d0, d1\n"

    "mov         r9, %2\n"

    "vst1.64     {d0}, [r9], %3\n"
    "vst1.64     {d1}, [r9]\n"

    "add         %0, #2\n"              // src += 2
    "add         %2, %3, lsl #1\n"      // dst += 2 * dst_stride
    "subs        %4,  #2\n"             // w   -= 2
    "beq         4f\n"

    // 1x8 block
    "3:\n"
    "vld1.8      {d0[0]}, [%0], %1\n"
    "vld1.8      {d0[1]}, [%0], %1\n"
    "vld1.8      {d0[2]}, [%0], %1\n"
    "vld1.8      {d0[3]}, [%0], %1\n"
    "vld1.8      {d0[4]}, [%0], %1\n"
    "vld1.8      {d0[5]}, [%0], %1\n"
    "vld1.8      {d0[6]}, [%0], %1\n"
    "vld1.8      {d0[7]}, [%0]\n"

    "vst1.64     {d0}, [%2]\n"

    "4:\n"

    : "+r"(src),              // %0
      "+r"(src_stride),       // %1
      "+r"(dst),              // %2
      "+r"(dst_stride),       // %3
      "+r"(width)             // %4
    : "r"(vtbl_4x4_transpose) // %5
    : "memory", "cc", "r9", "q0", "q1", "q2", "q3"
  );
}

void ReverseLineUV_NEON(const uint8* src,
                        uint8* dst_a, uint8* dst_b,
                        int width) {
  asm volatile(
    // compute where to start writing destination
    "add         %1, %3\n"            // dst_a + width
    "add         %2, %3\n"            // dst_b + width

    // work on input segments that are multiples of 16, but
    // width that has been passed is output segments, half
    // the size of input.
    "lsrs        r12, %3, #3\n"

    "beq         2f\n"

    // the output is written in to two blocks.
    "mov         r12, #-8\n"

    // back of destination by the size of the register that is
    // going to be reversed
    "sub         %1, #8\n"
    "sub         %2, #8\n"

    // the loop needs to run on blocks of 8.  what will be left
    // over is either a negative number, the residuals that need
    // to be done, or 0.  if this isn't subtracted off here the
    // loop will run one extra time.
    "sub         %3, #8\n"

    "1:\n"
      "vld2.8      {d0, d1}, [%0]!\n"           // src += 16

      // reverse the bytes in the 64 bit segments
      "vrev64.8    q0, q0\n"

      "vst1.8      {d0}, [%1], r12\n"           // dst_a -= 8
      "vst1.8      {d1}, [%2], r12\n"           // dst_b -= 8

      "subs        %3, #8\n"
      "bge         1b\n"

    // add 8 back to the counter.  if the result is 0 there is no
    // residuals so return
    "adds        %3, #8\n"
    "beq         4f\n"

    "add         %1, #8\n"
    "add         %2, #8\n"

    "2:\n"

    "mov         r12, #-1\n"

    "sub         %1, #1\n"
    "sub         %2, #1\n"

    "3:\n"
      "vld2.8      {d0[0], d1[0]}, [%0]!\n"       // src += 2

      "vst1.8      {d0[0]}, [%1], r12\n"          // dst_a -= 1
      "vst1.8      {d1[0]}, [%2], r12\n"          // dst_b -= 1

      "subs        %3, %3, #1\n"
      "bgt         3b\n"
    "4:\n"
    : "+r"(src),              // %0
      "+r"(dst_a),            // %1
      "+r"(dst_b),            // %2
      "+r"(width)             // %3
    :
    : "memory", "cc", "r12", "q0"
  );
}

static const uint8 vtbl_4x4_transpose_di[16] __attribute__((vector_size(16))) =
  { 0,  8,  1,  9,  2, 10,  3, 11,  4, 12,  5, 13,  6, 14,  7, 15 };

void TransposeUVWx8_NEON(const uint8* src, int src_stride,
                         uint8* dst_a, int dst_stride_a,
                         uint8* dst_b, int dst_stride_b,
                         int width) {
  asm volatile(
    // loops are on blocks of 8.  loop will stop when
    // counter gets to or below 0.  starting the counter
    // at w-8 allow for this
    "sub         %6, #8\n"

    // handle 8x8 blocks.  this should be the majority of the plane
    "1:\n"
      "mov         r9, %0\n"

      "vld2.8      {d0,  d1},  [r9], %1\n"
      "vld2.8      {d2,  d3},  [r9], %1\n"
      "vld2.8      {d4,  d5},  [r9], %1\n"
      "vld2.8      {d6,  d7},  [r9], %1\n"
      "vld2.8      {d16, d17}, [r9], %1\n"
      "vld2.8      {d18, d19}, [r9], %1\n"
      "vld2.8      {d20, d21}, [r9], %1\n"
      "vld2.8      {d22, d23}, [r9]\n"

      "vtrn.8      q1, q0\n"
      "vtrn.8      q3, q2\n"
      "vtrn.8      q9, q8\n"
      "vtrn.8      q11, q10\n"

      "vtrn.16     q1, q3\n"
      "vtrn.16     q0, q2\n"
      "vtrn.16     q9, q11\n"
      "vtrn.16     q8, q10\n"

      "vtrn.32     q1, q9\n"
      "vtrn.32     q0, q8\n"
      "vtrn.32     q3, q11\n"
      "vtrn.32     q2, q10\n"

      "vrev16.8    q0, q0\n"
      "vrev16.8    q1, q1\n"
      "vrev16.8    q2, q2\n"
      "vrev16.8    q3, q3\n"
      "vrev16.8    q8, q8\n"
      "vrev16.8    q9, q9\n"
      "vrev16.8    q10, q10\n"
      "vrev16.8    q11, q11\n"

      "mov         r9, %2\n"

      "vst1.8      {d2},  [r9], %3\n"
      "vst1.8      {d0},  [r9], %3\n"
      "vst1.8      {d6},  [r9], %3\n"
      "vst1.8      {d4},  [r9], %3\n"
      "vst1.8      {d18}, [r9], %3\n"
      "vst1.8      {d16}, [r9], %3\n"
      "vst1.8      {d22}, [r9], %3\n"
      "vst1.8      {d20}, [r9]\n"

      "mov         r9, %4\n"

      "vst1.8      {d3},  [r9], %5\n"
      "vst1.8      {d1},  [r9], %5\n"
      "vst1.8      {d7},  [r9], %5\n"
      "vst1.8      {d5},  [r9], %5\n"
      "vst1.8      {d19}, [r9], %5\n"
      "vst1.8      {d17}, [r9], %5\n"
      "vst1.8      {d23}, [r9], %5\n"
      "vst1.8      {d21}, [r9]\n"

      "add         %0, #8*2\n"            // src   += 8*2
      "add         %2, %3, lsl #3\n"      // dst_a += 8 * dst_stride_a
      "add         %4, %5, lsl #3\n"      // dst_b += 8 * dst_stride_b
      "subs        %6,  #8\n"             // w     -= 8
      "bge         1b\n"

    // add 8 back to counter.  if the result is 0 there are
    // no residuals.
    "adds        %6, #8\n"
    "beq         4f\n"

    // some residual, so between 1 and 7 lines left to transpose
    "cmp         %6, #2\n"
    "blt         3f\n"

    "cmp         %6, #4\n"
    "blt         2f\n"

    //TODO(frkoenig) : clean this up
    // 4x8 block
    "mov         r9, %0\n"
    "vld1.64     {d0}, [r9], %1\n"
    "vld1.64     {d1}, [r9], %1\n"
    "vld1.64     {d2}, [r9], %1\n"
    "vld1.64     {d3}, [r9], %1\n"
    "vld1.64     {d4}, [r9], %1\n"
    "vld1.64     {d5}, [r9], %1\n"
    "vld1.64     {d6}, [r9], %1\n"
    "vld1.64     {d7}, [r9]\n"

    "vld1.8      {q15}, [%7]\n"

    "vtrn.8      q0, q1\n"
    "vtrn.8      q2, q3\n"

    "vtbl.8      d16, {d0, d1}, d30\n"
    "vtbl.8      d17, {d0, d1}, d31\n"
    "vtbl.8      d18, {d2, d3}, d30\n"
    "vtbl.8      d19, {d2, d3}, d31\n"
    "vtbl.8      d20, {d4, d5}, d30\n"
    "vtbl.8      d21, {d4, d5}, d31\n"
    "vtbl.8      d22, {d6, d7}, d30\n"
    "vtbl.8      d23, {d6, d7}, d31\n"

    "mov         r9, %2\n"

    "vst1.32     {d16[0]},  [r9], %3\n"
    "vst1.32     {d16[1]},  [r9], %3\n"
    "vst1.32     {d17[0]},  [r9], %3\n"
    "vst1.32     {d17[1]},  [r9], %3\n"

    "add         r9, %2, #4\n"
    "vst1.32     {d20[0]}, [r9], %3\n"
    "vst1.32     {d20[1]}, [r9], %3\n"
    "vst1.32     {d21[0]}, [r9], %3\n"
    "vst1.32     {d21[1]}, [r9]\n"

    "mov         r9, %4\n"

    "vst1.32     {d18[0]}, [r9], %5\n"
    "vst1.32     {d18[1]}, [r9], %5\n"
    "vst1.32     {d19[0]}, [r9], %5\n"
    "vst1.32     {d19[1]}, [r9], %5\n"

    "add         r9, %4, #4\n"
    "vst1.32     {d22[0]},  [r9], %5\n"
    "vst1.32     {d22[1]},  [r9], %5\n"
    "vst1.32     {d23[0]},  [r9], %5\n"
    "vst1.32     {d23[1]},  [r9]\n"

    "add         %0, #4*2\n"            // src   += 4 * 2
    "add         %2, %3, lsl #2\n"      // dst_a += 4 * dst_stride_a
    "add         %4, %5, lsl #2\n"      // dst_b += 4 * dst_stride_b
    "subs        %6,  #4\n"             // w     -= 4
    "beq         4f\n"

    // some residual, check to see if it includes a 2x8 block,
    // or less
    "cmp         %6, #2\n"
    "blt         3f\n"

    // 2x8 block
    "2:\n"
    "mov         r9, %0\n"
    "vld2.16     {d0[0], d2[0]}, [r9], %1\n"
    "vld2.16     {d1[0], d3[0]}, [r9], %1\n"
    "vld2.16     {d0[1], d2[1]}, [r9], %1\n"
    "vld2.16     {d1[1], d3[1]}, [r9], %1\n"
    "vld2.16     {d0[2], d2[2]}, [r9], %1\n"
    "vld2.16     {d1[2], d3[2]}, [r9], %1\n"
    "vld2.16     {d0[3], d2[3]}, [r9], %1\n"
    "vld2.16     {d1[3], d3[3]}, [r9]\n"

    "vtrn.8      d0, d1\n"
    "vtrn.8      d2, d3\n"

    "mov         r9, %2\n"

    "vst1.64     {d0}, [r9], %3\n"
    "vst1.64     {d2}, [r9]\n"

    "mov         r9, %4\n"

    "vst1.64     {d1}, [r9], %5\n"
    "vst1.64     {d3}, [r9]\n"

    "add         %0, #2*2\n"            // src   += 2 * 2
    "add         %2, %3, lsl #1\n"      // dst_a += 2 * dst_stride_a
    "add         %4, %5, lsl #1\n"      // dst_b += 2 * dst_stride_b
    "subs        %6,  #2\n"             // w     -= 2
    "beq         4f\n"

    // 1x8 block
    "3:\n"
    "vld2.8      {d0[0], d1[0]}, [%0], %1\n"
    "vld2.8      {d0[1], d1[1]}, [%0], %1\n"
    "vld2.8      {d0[2], d1[2]}, [%0], %1\n"
    "vld2.8      {d0[3], d1[3]}, [%0], %1\n"
    "vld2.8      {d0[4], d1[4]}, [%0], %1\n"
    "vld2.8      {d0[5], d1[5]}, [%0], %1\n"
    "vld2.8      {d0[6], d1[6]}, [%0], %1\n"
    "vld2.8      {d0[7], d1[7]}, [%0]\n"

    "vst1.64     {d0}, [%2]\n"
    "vst1.64     {d1}, [%4]\n"

    "4:\n"

    : "+r"(src),              // %0
      "+r"(src_stride),       // %1
      "+r"(dst_a),            // %2
      "+r"(dst_stride_a),     // %3
      "+r"(dst_b),            // %4
      "+r"(dst_stride_b),     // %5
      "+r"(width)             // %6
    : "r"(vtbl_4x4_transpose_di)// %7
    : "memory", "cc", "r9",
      "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"
  );
}
#endif
}