test/aarch32-assembler.cc

// Copyright 2021 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

#include <xnnpack/aarch32-assembler.h>
#include <xnnpack/allocator.h>
#include <xnnpack/common.h>

#include <ios>

#include <gtest/gtest.h>

// clang-format off
#define EXPECT_INSTR(expected, actual)                                                                        \
  EXPECT_EQ(expected, actual) << "expected = 0x" << std::hex << std::setw(8) << std::setfill('0') << expected \
                              << std::endl << "  actual = 0x" << actual;
// clang-format on

#define CHECK_ENCODING(expected, call) \
  a.reset();                           \
  call;                                \
  EXPECT_INSTR(expected, *a.start())

#define EXPECT_ERROR(expected, call) \
  a.reset();                         \
  call;                              \
  EXPECT_EQ(expected, a.error());

namespace xnnpack {
namespace aarch32 {
TEST(AArch32Assembler, InstructionEncoding) {
  xnn_code_buffer b;
  xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
  Assembler a(&b);

  CHECK_ENCODING(0xE0810002, a.add(r0, r1, r2));
  CHECK_ENCODING(0xE28A9080, a.add(r9, r10, 128));

  CHECK_ENCODING(0xE12FFF1E, a.bx(lr));

  CHECK_ENCODING(0xE3500002, a.cmp(r0, 2));

  // Offset addressing mode.
  CHECK_ENCODING(0xE59D7060, a.ldr(r7, mem[sp, 96]));
  // Post-indexed addressing mode.
  CHECK_ENCODING(0xE490B000, a.ldr(r11, mem[r0], 0));
  CHECK_ENCODING(0xE490B060, a.ldr(r11, mem[r0], 96));
  // Offsets out of bounds.
  EXPECT_ERROR(Error::kInvalidOperand, a.ldr(r7, MemOperand(sp, 4096)));
  EXPECT_ERROR(Error::kInvalidOperand, a.ldr(r7, MemOperand(sp, -4096)));

  CHECK_ENCODING(0x31A0C003, a.movlo(r12, r3));
  CHECK_ENCODING(0x91A0A00C, a.movls(r10, r12));
  CHECK_ENCODING(0xE1A0A00C, a.mov(r10, r12));

  CHECK_ENCODING(0xE320F000, a.nop());

  CHECK_ENCODING(0xE8BD0FF0, a.pop({r4, r5, r6, r7, r8, r9, r10, r11}));
  EXPECT_ERROR(Error::kInvalidOperand, a.pop({}));
  EXPECT_ERROR(Error::kInvalidOperand, a.pop({r1}));

  CHECK_ENCODING(0xE92D0FF0, a.push({r4, r5, r6, r7, r8, r9, r10, r11}));
  EXPECT_ERROR(Error::kInvalidOperand, a.push({}));
  EXPECT_ERROR(Error::kInvalidOperand, a.push({r1}));

  CHECK_ENCODING(0xF5D3F000, a.pld(MemOperand(r3, 0)));
  CHECK_ENCODING(0xF5D3F040, a.pld(MemOperand(r3, 64)));

  CHECK_ENCODING(0xE0487002, a.sub(r7, r8, r2));
  CHECK_ENCODING(0xE2525010, a.subs(r5, r2, 16));

  CHECK_ENCODING(0xE315000F, a.tst(r5, 15));

  CHECK_ENCODING(0xF3FF8C4F, a.vdup_8(q12, d15[7]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vdup_8(q12, d15[8]));
  CHECK_ENCODING(0xF3FE8C4F, a.vdup_16(q12, d15[3]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vdup_16(q12, d15[4]));
  CHECK_ENCODING(0xF3FC8C4F, a.vdup_32(q12, d15[1]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vdup_32(q12, d15[2]));

  CHECK_ENCODING(0xF2BE04C6, a.vext_8(q0, q15, q3, 4));
  EXPECT_ERROR(Error::kInvalidOperand, a.vext_8(q0, q15, q3, 16));

  CHECK_ENCODING(0xF423070F, a.vld1_8({d0}, mem[r3]));
  CHECK_ENCODING(0xF423070D, a.vld1_8({d0}, mem[r3]++));

  CHECK_ENCODING(0xF42C178F, a.vld1_32({d1}, mem[r12]));
  CHECK_ENCODING(0xF42C178D, a.vld1_32({d1}, mem[r12]++));

  CHECK_ENCODING(0xF4A54CAF, a.vld1r_32({d4, d5}, mem[r5]));
  CHECK_ENCODING(0xF4A54CAD, a.vld1r_32({d4, d5}, mem[r5]++));

  CHECK_ENCODING(0xECF90B08, a.vldm(r9, {d16, d19}, true));
  CHECK_ENCODING(0xEC998B08, a.vldm(r9, {d8, d11}, false));
  CHECK_ENCODING(0xEC998B08, a.vldm(r9, {d8, d11}));
  CHECK_ENCODING(0xECB30A01, a.vldm(r3, {s0}, true));
  CHECK_ENCODING(0xEC930A01, a.vldm(r3, {s0}));

  CHECK_ENCODING(0xED99FB0E, a.vldr(d15, mem[r9, 56]));
  EXPECT_ERROR(Error::kInvalidOperand, a.vldr(d15, MemOperand(r9, 56, AddressingMode::kPostIndexed)));
  EXPECT_ERROR(Error::kInvalidOperand, a.vldr(d15, mem[r9, 256]));

  CHECK_ENCODING(0xF20E26C6, a.vmax_s8(q1, q15, q3));
  CHECK_ENCODING(0xF24ECFC4, a.vmax_f32(q14, q15, q2));

  CHECK_ENCODING(0xF20E26D6, a.vmin_s8(q1, q15, q3));
  CHECK_ENCODING(0xF220EFC6, a.vmin_f32(q7, q8, q3));

  CHECK_ENCODING(0xF3E80140, a.vmla_f32(q8, q4, d0[0]));
  CHECK_ENCODING(0xF3EC0160, a.vmla_f32(q8, q6, d0[1]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vmla_f32(q8, q4, d0[2]));

  CHECK_ENCODING(0xF2D9E246, a.vmlal_s16(q15, d9, d6[0]));
  CHECK_ENCODING(0xF2D8424A, a.vmlal_s16(q10, d8, d2[1]));
  CHECK_ENCODING(0xF2D88264, a.vmlal_s16(q12, d8, d4[2]));
  CHECK_ENCODING(0xF2D8626A, a.vmlal_s16(q11, d8, d2[3]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vmlal_s16(q15, d9, d6[4]));

  CHECK_ENCODING(0xEEB0EA4F, a.vmov(s28, s30));
  CHECK_ENCODING(0xF26101B1, a.vmov(d16, d17));
  CHECK_ENCODING(0xEC420B1F, a.vmov(d15, r0, r2));
  CHECK_ENCODING(0xF26041F0, a.vmov(q10, q8));

  CHECK_ENCODING(0xF2880A10, a.vmovl_s8(q0, d0));

  CHECK_ENCODING(0xECBD8B10, a.vpop({d8, d15}));

  CHECK_ENCODING(0xED2D4A08, a.vpush({s8, s15}));
  CHECK_ENCODING(0xED2DAA04, a.vpush({s20, s23}));
  CHECK_ENCODING(0xED2D8B10, a.vpush({d8, d15}));
  CHECK_ENCODING(0xED6D4B08, a.vpush({d20, d23}));

  CHECK_ENCODING(0xF25E00D2, a.vqadd_s16(q8, q15, q1));

  CHECK_ENCODING(0xF3A82CCE, a.vqdmulh_s32(q1, q12, d14[0]));
  CHECK_ENCODING(0xF3A82CEE, a.vqdmulh_s32(q1, q12, d14[1]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vqdmulh_s32(q1, q12, d14[2]));
  EXPECT_ERROR(Error::kInvalidOperand, a.vqdmulh_s32(q1, q12, d16[0]));

  CHECK_ENCODING(0xF3F602A0, a.vqmovn_s32(d16, q8));

  CHECK_ENCODING(0xF22C247E, a.vqshl_s32(q1, q15, q6));

  CHECK_ENCODING(0xF264C560, a.vrshl_s32(q14, q8, q2));

  CHECK_ENCODING(0xF40B070F, a.vst1_8({d0}, mem[r11]));
  CHECK_ENCODING(0xF40B070D, a.vst1_8({d0}, mem[r11]++));
  CHECK_ENCODING(0xF40B0707, a.vst1_8({d0}, mem[r11], r7));
  CHECK_ENCODING(0xF48B000F, a.vst1_8({d0[0]}, mem[r11]));
  CHECK_ENCODING(0xF48B00EF, a.vst1_8({d0[7]}, mem[r11]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vst1_8(d0[8], mem[r11]));

  CHECK_ENCODING(0xF40B074F, a.vst1_16({d0}, mem[r11]));
  CHECK_ENCODING(0xF40B074D, a.vst1_16({d0}, mem[r11]++));
  CHECK_ENCODING(0xF40B0747, a.vst1_16({d0}, mem[r11], r7));
  CHECK_ENCODING(0xF48B040F, a.vst1_16({d0[0]}, mem[r11]));
  CHECK_ENCODING(0xF48B04CF, a.vst1_16({d0[3]}, mem[r11]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vst1_16(d0[4], mem[r11]));

  CHECK_ENCODING(0xF44B0280, a.vst1_32({d16, d19}, mem[r11], r0));
  EXPECT_ERROR(Error::kInvalidRegisterListLength, a.vst1_32({d0, d4}, mem[r11], r0));
  EXPECT_ERROR(Error::kInvalidOperand, a.vst1_32({d16, d19}, mem[r11], sp));
  EXPECT_ERROR(Error::kInvalidOperand, a.vst1_32({d16, d19}, mem[r11], pc));
  CHECK_ENCODING(0xF404168F, a.vst1_32({d1, d3}, mem[r4]));
  CHECK_ENCODING(0xF44B0A8D, a.vst1_32({d16, d17}, mem[r11]++));
  CHECK_ENCODING(0xF4CB080F, a.vst1_32({d16[0]}, mem[r11]));
  // The surrounding braces are optional, but makes it look closer to native assembly.
  CHECK_ENCODING(0xF4CB080F, a.vst1_32(d16[0], mem[r11]));
  CHECK_ENCODING(0xF4CB088F, a.vst1_32(d16[1], mem[r11]));
  EXPECT_ERROR(Error::kInvalidLaneIndex, a.vst1_32(d16[2], mem[r11]));
  CHECK_ENCODING(0xF4C6C80D, a.vst1_32({d28[0]}, mem[r6]++));

  ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
}

TEST(AArch32Assembler, Label) {
  xnn_code_buffer b;
  xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
  Assembler a(&b);

  Label l1;
  a.add(r0, r0, r0);

  // Branch to unbound label.
  auto b1 = a.offset();
  a.beq(l1);

  a.add(r1, r1, r1);

  auto b2 = a.offset();
  a.bne(l1);

  a.add(r2, r2, r2);

  a.bind(l1);

  // Check that b1 and b2 are both patched after binding l1.
  EXPECT_INSTR(0x0A000002, *b1);
  EXPECT_INSTR(0x1A000000, *b2);

  a.add(r0, r1, r2);

  // Branch to bound label.
  auto b3 = a.offset();
  a.bhi(l1);
  auto b4 = a.offset();
  a.bhs(l1);
  auto b5 = a.offset();
  a.blo(l1);
  auto b6 = a.offset();
  a.b(l1);

  EXPECT_INSTR(0x8AFFFFFD, *b3);
  EXPECT_INSTR(0x2AFFFFFC, *b4);
  EXPECT_INSTR(0x3AFFFFFB, *b5);
  EXPECT_INSTR(0xEAFFFFFA, *b6);

  // Binding a bound label is an error.
  a.bind(l1);
  EXPECT_ERROR(Error::kLabelAlreadyBound, a.bind(l1));

  // Check for bind failure due to too many users of label.
  Label lfail;
  a.reset();
  // Arbitrary high number of users that we probably won't support.
  for (int i = 0; i < 1000; i++) {
    a.beq(lfail);
  }
  EXPECT_EQ(Error::kLabelHasTooManyUsers, a.error());

  ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
}

TEST(AArch32Assembler, Align) {
  xnn_code_buffer b;
  xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);
  Assembler a(&b);

  a.add(r0, r1, r2);
  a.align(4);
  EXPECT_EQ(0, reinterpret_cast<uintptr_t>(a.offset()) & 0x3);
  EXPECT_EQ(4, a.code_size_in_bytes());

  a.align(8);
  EXPECT_EQ(0, reinterpret_cast<uintptr_t>(a.offset()) & 0x7);
  EXPECT_EQ(8, a.code_size_in_bytes());

  a.add(r0, r1, r2);
  a.align(8);
  EXPECT_EQ(0, reinterpret_cast<uintptr_t>(a.offset()) & 0x7);
  EXPECT_EQ(16, a.code_size_in_bytes());

  a.add(r0, r1, r2);
  EXPECT_EQ(20, a.code_size_in_bytes());

  a.align(16);
  EXPECT_EQ(0, reinterpret_cast<uintptr_t>(a.offset()) & 0xF);
  EXPECT_EQ(32, a.code_size_in_bytes());

  a.add(r0, r1, r2);
  a.add(r0, r1, r2);
  EXPECT_EQ(40, a.code_size_in_bytes());

  a.align(16);
  EXPECT_EQ(0, reinterpret_cast<uintptr_t>(a.offset()) & 0xF);
  EXPECT_EQ(48, a.code_size_in_bytes());

  // Not power-of-two.
  EXPECT_ERROR(Error::kInvalidOperand, a.align(6));
  // Is power-of-two but is not a multiple of instruction size.
  EXPECT_ERROR(Error::kInvalidOperand, a.align(2));

  ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
}

TEST(AArch32Assembler, CoreRegisterList) {
  EXPECT_EQ(0x3, CoreRegisterList({r0, r1}));
  EXPECT_EQ(0xFC00, CoreRegisterList({r10, r11, r12, r13, r14, r15}));

  EXPECT_FALSE(CoreRegisterList({}).has_more_than_one_register());
  EXPECT_FALSE(CoreRegisterList({r0}).has_more_than_one_register());
  EXPECT_FALSE(CoreRegisterList({r1}).has_more_than_one_register());
  EXPECT_TRUE(CoreRegisterList({r0, r1}).has_more_than_one_register());
}

TEST(AArch32Assembler, ConsecutiveRegisterList) {
  SRegisterList s1 = SRegisterList(s0, s9);
  EXPECT_EQ(s1.start, s0);
  EXPECT_EQ(s1.length, 10);

  DRegisterList d1 = DRegisterList(d4, d5);
  EXPECT_EQ(d1.start, d4);
  EXPECT_EQ(d1.length, 2);
}

TEST(AArch32Assembler, MemOperand) {
  EXPECT_EQ(MemOperand(r0, 4, AddressingMode::kOffset), (mem[r0, 4]));
}

TEST(AArch32Assembler, DRegisterLane) {
  EXPECT_EQ((DRegisterLane{2, 0}), d2[0]);
  EXPECT_EQ((DRegisterLane{2, 1}), d2[1]);
}

TEST(AArch32Assembler, CodeBufferOverflow) {
  xnn_code_buffer b;
  xnn_allocate_code_memory(&b, 4);
  Assembler a(&b);
  a.add(r0, r0, 2);
  EXPECT_EQ(Error::kNoError, a.error());

  a.bx(lr);
  EXPECT_EQ(Error::kOutOfMemory, a.error());

  ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
}

TEST(AArch32Assembler, AllocateAndRelease) {
  xnn_code_buffer b;
  ASSERT_EQ(xnn_status_success, xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE));
  ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
}

#if XNN_ARCH_ARM
TEST(AArch32Assembler, JitAllocCodeBuffer) {
  typedef uint32_t (*Func)(uint32_t);

  xnn_code_buffer b;
  xnn_allocate_code_memory(&b, XNN_DEFAULT_CODE_BUFFER_SIZE);

  Assembler a(&b);
  a.add(r0, r0, 2).bx(lr);

  Func fn = reinterpret_cast<Func>(a.finalize());

  ASSERT_EQ(3, fn(1));

  ASSERT_EQ(xnn_status_success, xnn_release_code_memory(&b));
}
#endif
}  // namespace aarch32
}  // namespace xnnpack