Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1 | //===- subzero/src/IceTargetLoweringARM32.cpp - ARM32 lowering ------------===// |
| 2 | // |
| 3 | // The Subzero Code Generator |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
Andrew Scull | 9612d32 | 2015-07-06 14:53:25 -0700 | [diff] [blame] | 9 | /// |
| 10 | /// \file |
Jim Stichnoth | 92a6e5b | 2015-12-02 16:52:44 -0800 | [diff] [blame] | 11 | /// \brief Implements the TargetLoweringARM32 class, which consists almost |
Andrew Scull | 9612d32 | 2015-07-06 14:53:25 -0700 | [diff] [blame] | 12 | /// entirely of the lowering sequence for each high-level instruction. |
| 13 | /// |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 14 | //===----------------------------------------------------------------------===// |
John Porto | 67f8de9 | 2015-06-25 10:14:17 -0700 | [diff] [blame] | 15 | #include "IceTargetLoweringARM32.h" |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 16 | |
| 17 | #include "IceCfg.h" |
| 18 | #include "IceCfgNode.h" |
| 19 | #include "IceClFlags.h" |
| 20 | #include "IceDefs.h" |
| 21 | #include "IceELFObjectWriter.h" |
| 22 | #include "IceGlobalInits.h" |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 23 | #include "IceInstARM32.def" |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 24 | #include "IceInstARM32.h" |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 25 | #include "IceInstVarIter.h" |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 26 | #include "IceLiveness.h" |
| 27 | #include "IceOperand.h" |
Jan Voung | 5348369 | 2015-07-16 10:47:46 -0700 | [diff] [blame] | 28 | #include "IcePhiLoweringImpl.h" |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 29 | #include "IceRegistersARM32.h" |
| 30 | #include "IceTargetLoweringARM32.def" |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 31 | #include "IceUtils.h" |
John Porto | 67f8de9 | 2015-06-25 10:14:17 -0700 | [diff] [blame] | 32 | #include "llvm/Support/MathExtras.h" |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 33 | |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 34 | #include <algorithm> |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 35 | #include <array> |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 36 | #include <utility> |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 37 | |
John Porto | 53611e2 | 2015-12-30 07:30:10 -0800 | [diff] [blame] | 38 | namespace ARM32 { |
| 39 | std::unique_ptr<::Ice::TargetLowering> createTargetLowering(::Ice::Cfg *Func) { |
John Porto | 4a56686 | 2016-01-04 09:33:41 -0800 | [diff] [blame] | 40 | return ::Ice::ARM32::TargetARM32::create(Func); |
John Porto | 53611e2 | 2015-12-30 07:30:10 -0800 | [diff] [blame] | 41 | } |
| 42 | |
| 43 | std::unique_ptr<::Ice::TargetDataLowering> |
| 44 | createTargetDataLowering(::Ice::GlobalContext *Ctx) { |
John Porto | 4a56686 | 2016-01-04 09:33:41 -0800 | [diff] [blame] | 45 | return ::Ice::ARM32::TargetDataARM32::create(Ctx); |
John Porto | 53611e2 | 2015-12-30 07:30:10 -0800 | [diff] [blame] | 46 | } |
| 47 | |
| 48 | std::unique_ptr<::Ice::TargetHeaderLowering> |
| 49 | createTargetHeaderLowering(::Ice::GlobalContext *Ctx) { |
John Porto | 4a56686 | 2016-01-04 09:33:41 -0800 | [diff] [blame] | 50 | return ::Ice::ARM32::TargetHeaderARM32::create(Ctx); |
John Porto | 53611e2 | 2015-12-30 07:30:10 -0800 | [diff] [blame] | 51 | } |
| 52 | |
Karl Schimpf | 5403f5d | 2016-01-15 11:07:46 -0800 | [diff] [blame] | 53 | void staticInit(::Ice::GlobalContext *Ctx) { |
| 54 | ::Ice::ARM32::TargetARM32::staticInit(Ctx); |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 55 | if (Ice::getFlags().getUseNonsfi()) { |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 56 | // In nonsfi, we need to reference the _GLOBAL_OFFSET_TABLE_ for accessing |
| 57 | // globals. The GOT is an external symbol (i.e., it is not defined in the |
| 58 | // pexe) so we need to register it as such so that ELF emission won't barf |
| 59 | // on an "unknown" symbol. The GOT is added to the External symbols list |
| 60 | // here because staticInit() is invoked in a single-thread context. |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 61 | Ctx->getConstantExternSym(Ctx->getGlobalString(::Ice::GlobalOffsetTable)); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 62 | } |
Jim Stichnoth | 8ff4b28 | 2016-01-04 15:39:06 -0800 | [diff] [blame] | 63 | } |
Karl Schimpf | 57ec7df | 2016-01-15 08:11:00 -0800 | [diff] [blame] | 64 | |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 65 | bool shouldBePooled(const ::Ice::Constant *C) { |
| 66 | return ::Ice::ARM32::TargetARM32::shouldBePooled(C); |
| 67 | } |
| 68 | |
John Porto | 53611e2 | 2015-12-30 07:30:10 -0800 | [diff] [blame] | 69 | } // end of namespace ARM32 |
| 70 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 71 | namespace Ice { |
John Porto | 4a56686 | 2016-01-04 09:33:41 -0800 | [diff] [blame] | 72 | namespace ARM32 { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 73 | |
Karl Schimpf | 57ec7df | 2016-01-15 08:11:00 -0800 | [diff] [blame] | 74 | namespace { |
| 75 | |
| 76 | /// SizeOf is used to obtain the size of an initializer list as a constexpr |
| 77 | /// expression. This is only needed until our C++ library is updated to |
| 78 | /// C++ 14 -- which defines constexpr members to std::initializer_list. |
| 79 | class SizeOf { |
| 80 | SizeOf(const SizeOf &) = delete; |
| 81 | SizeOf &operator=(const SizeOf &) = delete; |
| 82 | |
| 83 | public: |
| 84 | constexpr SizeOf() : Size(0) {} |
| 85 | template <typename... T> |
| 86 | explicit constexpr SizeOf(T...) |
| 87 | : Size(__length<T...>::value) {} |
| 88 | constexpr SizeT size() const { return Size; } |
| 89 | |
| 90 | private: |
| 91 | template <typename T, typename... U> struct __length { |
| 92 | static constexpr std::size_t value = 1 + __length<U...>::value; |
| 93 | }; |
| 94 | |
| 95 | template <typename T> struct __length<T> { |
| 96 | static constexpr std::size_t value = 1; |
| 97 | }; |
| 98 | |
| 99 | const std::size_t Size; |
| 100 | }; |
| 101 | |
| 102 | } // end of anonymous namespace |
| 103 | |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 104 | // Defines the RegARM32::Table table with register information. |
Karl Schimpf | 57ec7df | 2016-01-15 08:11:00 -0800 | [diff] [blame] | 105 | RegARM32::RegTableType RegARM32::RegTable[RegARM32::Reg_NUM] = { |
| 106 | #define X(val, encode, name, cc_arg, scratch, preserved, stackptr, frameptr, \ |
| 107 | isGPR, isInt, isI64Pair, isFP32, isFP64, isVec128, alias_init) \ |
| 108 | { \ |
| 109 | name, encode, cc_arg, scratch, preserved, stackptr, frameptr, isGPR, \ |
| 110 | isInt, isI64Pair, isFP32, isFP64, isVec128, \ |
| 111 | (SizeOf alias_init).size(), alias_init \ |
| 112 | } \ |
| 113 | , |
| 114 | REGARM32_TABLE |
| 115 | #undef X |
| 116 | }; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 117 | |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 118 | namespace { |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 119 | |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 120 | // The following table summarizes the logic for lowering the icmp instruction |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 121 | // for i32 and narrower types. Each icmp condition has a clear mapping to an |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 122 | // ARM32 conditional move instruction. |
| 123 | |
| 124 | const struct TableIcmp32_ { |
| 125 | CondARM32::Cond Mapping; |
| 126 | } TableIcmp32[] = { |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 127 | #define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V) \ |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 128 | { CondARM32::C_32 } \ |
| 129 | , |
| 130 | ICMPARM32_TABLE |
| 131 | #undef X |
| 132 | }; |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 133 | |
| 134 | // The following table summarizes the logic for lowering the icmp instruction |
| 135 | // for the i64 type. Two conditional moves are needed for setting to 1 or 0. |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 136 | // The operands may need to be swapped, and there is a slight difference for |
| 137 | // signed vs unsigned (comparing hi vs lo first, and using cmp vs sbc). |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 138 | const struct TableIcmp64_ { |
| 139 | bool IsSigned; |
| 140 | bool Swapped; |
| 141 | CondARM32::Cond C1, C2; |
| 142 | } TableIcmp64[] = { |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 143 | #define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V) \ |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 144 | { is_signed, swapped64, CondARM32::C1_64, CondARM32::C2_64 } \ |
| 145 | , |
| 146 | ICMPARM32_TABLE |
| 147 | #undef X |
| 148 | }; |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 149 | |
| 150 | CondARM32::Cond getIcmp32Mapping(InstIcmp::ICond Cond) { |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 151 | assert(Cond < llvm::array_lengthof(TableIcmp32)); |
| 152 | return TableIcmp32[Cond].Mapping; |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 153 | } |
| 154 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 155 | // In some cases, there are x-macros tables for both high-level and low-level |
| 156 | // instructions/operands that use the same enum key value. The tables are kept |
| 157 | // separate to maintain a proper separation between abstraction layers. There |
| 158 | // is a risk that the tables could get out of sync if enum values are reordered |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 159 | // or if entries are added or deleted. The following anonymous namespaces use |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 160 | // static_asserts to ensure everything is kept in sync. |
| 161 | |
| 162 | // Validate the enum values in ICMPARM32_TABLE. |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 163 | namespace { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 164 | // Define a temporary set of enum values based on low-level table entries. |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 165 | enum _icmp_ll_enum { |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 166 | #define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V) \ |
| 167 | _icmp_ll_##val, |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 168 | ICMPARM32_TABLE |
| 169 | #undef X |
| 170 | _num |
| 171 | }; |
| 172 | // Define a set of constants based on high-level table entries. |
Manasij Mukherjee | 0c70417 | 2016-07-21 12:40:24 -0700 | [diff] [blame^] | 173 | #define X(tag, reverse, str) \ |
| 174 | static constexpr int _icmp_hl_##tag = InstIcmp::tag; |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 175 | ICEINSTICMP_TABLE |
| 176 | #undef X |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 177 | // Define a set of constants based on low-level table entries, and ensure the |
| 178 | // table entry keys are consistent. |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 179 | #define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V) \ |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 180 | static_assert( \ |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 181 | _icmp_ll_##val == _icmp_hl_##val, \ |
| 182 | "Inconsistency between ICMPARM32_TABLE and ICEINSTICMP_TABLE: " #val); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 183 | ICMPARM32_TABLE |
| 184 | #undef X |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 185 | // Repeat the static asserts with respect to the high-level table entries in |
| 186 | // case the high-level table has extra entries. |
Manasij Mukherjee | 0c70417 | 2016-07-21 12:40:24 -0700 | [diff] [blame^] | 187 | #define X(tag, reverse, str) \ |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 188 | static_assert( \ |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 189 | _icmp_hl_##tag == _icmp_ll_##tag, \ |
| 190 | "Inconsistency between ICMPARM32_TABLE and ICEINSTICMP_TABLE: " #tag); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 191 | ICEINSTICMP_TABLE |
| 192 | #undef X |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 193 | } // end of anonymous namespace |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 194 | |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 195 | // Stack alignment |
| 196 | const uint32_t ARM32_STACK_ALIGNMENT_BYTES = 16; |
| 197 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 198 | // Value is in bytes. Return Value adjusted to the next highest multiple of the |
| 199 | // stack alignment. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 200 | uint32_t applyStackAlignment(uint32_t Value) { |
| 201 | return Utils::applyAlignment(Value, ARM32_STACK_ALIGNMENT_BYTES); |
| 202 | } |
| 203 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 204 | // Value is in bytes. Return Value adjusted to the next highest multiple of the |
| 205 | // stack alignment required for the given type. |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 206 | uint32_t applyStackAlignmentTy(uint32_t Value, Type Ty) { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 207 | // Use natural alignment, except that normally (non-NaCl) ARM only aligns |
| 208 | // vectors to 8 bytes. |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 209 | // TODO(jvoung): Check this ... |
| 210 | size_t typeAlignInBytes = typeWidthInBytes(Ty); |
| 211 | if (isVectorType(Ty)) |
| 212 | typeAlignInBytes = 8; |
| 213 | return Utils::applyAlignment(Value, typeAlignInBytes); |
| 214 | } |
| 215 | |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 216 | // Conservatively check if at compile time we know that the operand is |
| 217 | // definitely a non-zero integer. |
| 218 | bool isGuaranteedNonzeroInt(const Operand *Op) { |
| 219 | if (auto *Const = llvm::dyn_cast_or_null<ConstantInteger32>(Op)) { |
| 220 | return Const->getValue() != 0; |
| 221 | } |
| 222 | return false; |
| 223 | } |
| 224 | |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 225 | } // end of anonymous namespace |
| 226 | |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 227 | TargetARM32Features::TargetARM32Features(const ClFlags &Flags) { |
Jan Voung | d062f73 | 2015-06-15 17:17:31 -0700 | [diff] [blame] | 228 | static_assert( |
| 229 | (ARM32InstructionSet::End - ARM32InstructionSet::Begin) == |
| 230 | (TargetInstructionSet::ARM32InstructionSet_End - |
| 231 | TargetInstructionSet::ARM32InstructionSet_Begin), |
| 232 | "ARM32InstructionSet range different from TargetInstructionSet"); |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 233 | if (Flags.getTargetInstructionSet() != |
Jan Voung | d062f73 | 2015-06-15 17:17:31 -0700 | [diff] [blame] | 234 | TargetInstructionSet::BaseInstructionSet) { |
| 235 | InstructionSet = static_cast<ARM32InstructionSet>( |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 236 | (Flags.getTargetInstructionSet() - |
Jan Voung | d062f73 | 2015-06-15 17:17:31 -0700 | [diff] [blame] | 237 | TargetInstructionSet::ARM32InstructionSet_Begin) + |
| 238 | ARM32InstructionSet::Begin); |
| 239 | } |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 240 | } |
| 241 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 242 | namespace { |
| 243 | constexpr SizeT NumGPRArgs = |
| 244 | #define X(val, encode, name, cc_arg, scratch, preserved, stackptr, frameptr, \ |
John Porto | dff7dbd | 2016-01-04 09:49:55 -0800 | [diff] [blame] | 245 | isGPR, isInt, isI64Pair, isFP32, isFP64, isVec128, alias_init) \ |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 246 | +(((cc_arg) > 0) ? 1 : 0) |
| 247 | REGARM32_GPR_TABLE |
| 248 | #undef X |
| 249 | ; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 250 | std::array<RegNumT, NumGPRArgs> GPRArgInitializer; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 251 | |
| 252 | constexpr SizeT NumI64Args = |
| 253 | #define X(val, encode, name, cc_arg, scratch, preserved, stackptr, frameptr, \ |
John Porto | dff7dbd | 2016-01-04 09:49:55 -0800 | [diff] [blame] | 254 | isGPR, isInt, isI64Pair, isFP32, isFP64, isVec128, alias_init) \ |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 255 | +(((cc_arg) > 0) ? 1 : 0) |
| 256 | REGARM32_I64PAIR_TABLE |
| 257 | #undef X |
| 258 | ; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 259 | std::array<RegNumT, NumI64Args> I64ArgInitializer; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 260 | |
| 261 | constexpr SizeT NumFP32Args = |
| 262 | #define X(val, encode, name, cc_arg, scratch, preserved, stackptr, frameptr, \ |
John Porto | dff7dbd | 2016-01-04 09:49:55 -0800 | [diff] [blame] | 263 | isGPR, isInt, isI64Pair, isFP32, isFP64, isVec128, alias_init) \ |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 264 | +(((cc_arg) > 0) ? 1 : 0) |
| 265 | REGARM32_FP32_TABLE |
| 266 | #undef X |
| 267 | ; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 268 | std::array<RegNumT, NumFP32Args> FP32ArgInitializer; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 269 | |
| 270 | constexpr SizeT NumFP64Args = |
| 271 | #define X(val, encode, name, cc_arg, scratch, preserved, stackptr, frameptr, \ |
John Porto | dff7dbd | 2016-01-04 09:49:55 -0800 | [diff] [blame] | 272 | isGPR, isInt, isI64Pair, isFP32, isFP64, isVec128, alias_init) \ |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 273 | +(((cc_arg) > 0) ? 1 : 0) |
| 274 | REGARM32_FP64_TABLE |
| 275 | #undef X |
| 276 | ; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 277 | std::array<RegNumT, NumFP64Args> FP64ArgInitializer; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 278 | |
| 279 | constexpr SizeT NumVec128Args = |
| 280 | #define X(val, encode, name, cc_arg, scratch, preserved, stackptr, frameptr, \ |
John Porto | dff7dbd | 2016-01-04 09:49:55 -0800 | [diff] [blame] | 281 | isGPR, isInt, isI64Pair, isFP32, isFP64, isVec128, alias_init) \ |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 282 | +(((cc_arg > 0)) ? 1 : 0) |
| 283 | REGARM32_VEC128_TABLE |
| 284 | #undef X |
| 285 | ; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 286 | std::array<RegNumT, NumVec128Args> Vec128ArgInitializer; |
Jim Stichnoth | 2544d4d | 2016-01-22 13:07:46 -0800 | [diff] [blame] | 287 | |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 288 | const char *getRegClassName(RegClass C) { |
Jim Stichnoth | 2544d4d | 2016-01-22 13:07:46 -0800 | [diff] [blame] | 289 | auto ClassNum = static_cast<RegARM32::RegClassARM32>(C); |
| 290 | assert(ClassNum < RegARM32::RCARM32_NUM); |
| 291 | switch (ClassNum) { |
| 292 | default: |
| 293 | assert(C < RC_Target); |
| 294 | return regClassString(C); |
Karl Schimpf | e54e530 | 2016-02-10 13:38:10 -0800 | [diff] [blame] | 295 | // Add handling of new register classes below. |
| 296 | case RegARM32::RCARM32_QtoS: |
| 297 | return "QtoS"; |
Jim Stichnoth | 2544d4d | 2016-01-22 13:07:46 -0800 | [diff] [blame] | 298 | } |
| 299 | } |
| 300 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 301 | } // end of anonymous namespace |
| 302 | |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 303 | TargetARM32::TargetARM32(Cfg *Func) |
John Porto | ac2388c | 2016-01-22 07:10:56 -0800 | [diff] [blame] | 304 | : TargetLowering(Func), NeedSandboxing(SandboxingType == ST_NaCl), |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 305 | CPUFeatures(getFlags()) {} |
Jim Stichnoth | 94844f1 | 2015-11-04 16:06:16 -0800 | [diff] [blame] | 306 | |
Karl Schimpf | 5403f5d | 2016-01-15 11:07:46 -0800 | [diff] [blame] | 307 | void TargetARM32::staticInit(GlobalContext *Ctx) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 308 | RegNumT::setLimit(RegARM32::Reg_NUM); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 309 | // Limit this size (or do all bitsets need to be the same width)??? |
John Porto | e82b560 | 2016-02-24 15:58:55 -0800 | [diff] [blame] | 310 | SmallBitVector IntegerRegisters(RegARM32::Reg_NUM); |
| 311 | SmallBitVector I64PairRegisters(RegARM32::Reg_NUM); |
| 312 | SmallBitVector Float32Registers(RegARM32::Reg_NUM); |
| 313 | SmallBitVector Float64Registers(RegARM32::Reg_NUM); |
| 314 | SmallBitVector VectorRegisters(RegARM32::Reg_NUM); |
| 315 | SmallBitVector QtoSRegisters(RegARM32::Reg_NUM); |
| 316 | SmallBitVector InvalidRegisters(RegARM32::Reg_NUM); |
Eric Holk | 658bae2 | 2016-02-08 15:22:18 -0800 | [diff] [blame] | 317 | const unsigned EncodedReg_q8 = RegARM32::RegTable[RegARM32::Reg_q8].Encoding; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 318 | for (int i = 0; i < RegARM32::Reg_NUM; ++i) { |
Karl Schimpf | 57ec7df | 2016-01-15 08:11:00 -0800 | [diff] [blame] | 319 | const auto &Entry = RegARM32::RegTable[i]; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 320 | IntegerRegisters[i] = Entry.IsInt; |
| 321 | I64PairRegisters[i] = Entry.IsI64Pair; |
| 322 | Float32Registers[i] = Entry.IsFP32; |
| 323 | Float64Registers[i] = Entry.IsFP64; |
| 324 | VectorRegisters[i] = Entry.IsVec128; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 325 | RegisterAliases[i].resize(RegARM32::Reg_NUM); |
Eric Holk | 658bae2 | 2016-02-08 15:22:18 -0800 | [diff] [blame] | 326 | // TODO(eholk): It would be better to store a QtoS flag in the |
| 327 | // IceRegistersARM32 table than to compare their encodings here. |
| 328 | QtoSRegisters[i] = Entry.IsVec128 && Entry.Encoding < EncodedReg_q8; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 329 | for (int j = 0; j < Entry.NumAliases; ++j) { |
| 330 | assert(i == j || !RegisterAliases[i][Entry.Aliases[j]]); |
| 331 | RegisterAliases[i].set(Entry.Aliases[j]); |
| 332 | } |
| 333 | assert(RegisterAliases[i][i]); |
| 334 | if (Entry.CCArg <= 0) { |
| 335 | continue; |
| 336 | } |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 337 | const auto RegNum = RegNumT::fromInt(i); |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 338 | if (Entry.IsGPR) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 339 | GPRArgInitializer[Entry.CCArg - 1] = RegNum; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 340 | } else if (Entry.IsI64Pair) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 341 | I64ArgInitializer[Entry.CCArg - 1] = RegNum; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 342 | } else if (Entry.IsFP32) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 343 | FP32ArgInitializer[Entry.CCArg - 1] = RegNum; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 344 | } else if (Entry.IsFP64) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 345 | FP64ArgInitializer[Entry.CCArg - 1] = RegNum; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 346 | } else if (Entry.IsVec128) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 347 | Vec128ArgInitializer[Entry.CCArg - 1] = RegNum; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 348 | } |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 349 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 350 | TypeToRegisterSet[IceType_void] = InvalidRegisters; |
| 351 | TypeToRegisterSet[IceType_i1] = IntegerRegisters; |
| 352 | TypeToRegisterSet[IceType_i8] = IntegerRegisters; |
| 353 | TypeToRegisterSet[IceType_i16] = IntegerRegisters; |
| 354 | TypeToRegisterSet[IceType_i32] = IntegerRegisters; |
John Porto | ed2c06b | 2015-10-01 15:27:15 -0700 | [diff] [blame] | 355 | TypeToRegisterSet[IceType_i64] = I64PairRegisters; |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 356 | TypeToRegisterSet[IceType_f32] = Float32Registers; |
| 357 | TypeToRegisterSet[IceType_f64] = Float64Registers; |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 358 | TypeToRegisterSet[IceType_v4i1] = VectorRegisters; |
| 359 | TypeToRegisterSet[IceType_v8i1] = VectorRegisters; |
| 360 | TypeToRegisterSet[IceType_v16i1] = VectorRegisters; |
| 361 | TypeToRegisterSet[IceType_v16i8] = VectorRegisters; |
| 362 | TypeToRegisterSet[IceType_v8i16] = VectorRegisters; |
| 363 | TypeToRegisterSet[IceType_v4i32] = VectorRegisters; |
| 364 | TypeToRegisterSet[IceType_v4f32] = VectorRegisters; |
Eric Holk | 658bae2 | 2016-02-08 15:22:18 -0800 | [diff] [blame] | 365 | TypeToRegisterSet[RegARM32::RCARM32_QtoS] = QtoSRegisters; |
Karl Schimpf | 5403f5d | 2016-01-15 11:07:46 -0800 | [diff] [blame] | 366 | |
Jim Stichnoth | b40595a | 2016-01-29 06:14:31 -0800 | [diff] [blame] | 367 | for (size_t i = 0; i < llvm::array_lengthof(TypeToRegisterSet); ++i) |
| 368 | TypeToRegisterSetUnfiltered[i] = TypeToRegisterSet[i]; |
| 369 | |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 370 | filterTypeToRegisterSet(Ctx, RegARM32::Reg_NUM, TypeToRegisterSet, |
| 371 | llvm::array_lengthof(TypeToRegisterSet), |
| 372 | [](RegNumT RegNum) -> std::string { |
| 373 | // This function simply removes ", " from the |
| 374 | // register name. |
| 375 | std::string Name = RegARM32::getRegName(RegNum); |
| 376 | constexpr const char RegSeparator[] = ", "; |
| 377 | constexpr size_t RegSeparatorWidth = |
| 378 | llvm::array_lengthof(RegSeparator) - 1; |
| 379 | for (size_t Pos = Name.find(RegSeparator); |
| 380 | Pos != std::string::npos; |
| 381 | Pos = Name.find(RegSeparator)) { |
| 382 | Name.replace(Pos, RegSeparatorWidth, ""); |
| 383 | } |
| 384 | return Name; |
| 385 | }, |
| 386 | getRegClassName); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 387 | } |
| 388 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 389 | namespace { |
| 390 | void copyRegAllocFromInfWeightVariable64On32(const VarList &Vars) { |
| 391 | for (Variable *Var : Vars) { |
| 392 | auto *Var64 = llvm::dyn_cast<Variable64On32>(Var); |
| 393 | if (!Var64) { |
| 394 | // This is not the variable we are looking for. |
| 395 | continue; |
| 396 | } |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 397 | // only allow infinite-weight i64 temporaries to be register allocated. |
| 398 | assert(!Var64->hasReg() || Var64->mustHaveReg()); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 399 | if (!Var64->hasReg()) { |
| 400 | continue; |
| 401 | } |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 402 | const auto FirstReg = |
| 403 | RegNumT::fixme(RegARM32::getI64PairFirstGPRNum(Var->getRegNum())); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 404 | // This assumes little endian. |
| 405 | Variable *Lo = Var64->getLo(); |
| 406 | Variable *Hi = Var64->getHi(); |
| 407 | assert(Lo->hasReg() == Hi->hasReg()); |
| 408 | if (Lo->hasReg()) { |
| 409 | continue; |
| 410 | } |
| 411 | Lo->setRegNum(FirstReg); |
| 412 | Lo->setMustHaveReg(); |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 413 | Hi->setRegNum(RegNumT::fixme(FirstReg + 1)); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 414 | Hi->setMustHaveReg(); |
| 415 | } |
| 416 | } |
| 417 | } // end of anonymous namespace |
| 418 | |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 419 | uint32_t TargetARM32::getCallStackArgumentsSizeBytes(const InstCall *Call) { |
| 420 | TargetARM32::CallingConv CC; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 421 | RegNumT DummyReg; |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 422 | size_t OutArgsSizeBytes = 0; |
| 423 | for (SizeT i = 0, NumArgs = Call->getNumArgs(); i < NumArgs; ++i) { |
| 424 | Operand *Arg = legalizeUndef(Call->getArg(i)); |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 425 | const Type Ty = Arg->getType(); |
| 426 | if (isScalarIntegerType(Ty)) { |
| 427 | if (CC.argInGPR(Ty, &DummyReg)) { |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 428 | continue; |
| 429 | } |
| 430 | } else { |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 431 | if (CC.argInVFP(Ty, &DummyReg)) { |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 432 | continue; |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | OutArgsSizeBytes = applyStackAlignmentTy(OutArgsSizeBytes, Ty); |
| 437 | OutArgsSizeBytes += typeWidthInBytesOnStack(Ty); |
| 438 | } |
| 439 | |
| 440 | return applyStackAlignment(OutArgsSizeBytes); |
| 441 | } |
| 442 | |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 443 | void TargetARM32::genTargetHelperCallFor(Inst *Instr) { |
| 444 | constexpr bool NoTailCall = false; |
| 445 | constexpr bool IsTargetHelperCall = true; |
| 446 | |
| 447 | switch (Instr->getKind()) { |
| 448 | default: |
| 449 | return; |
| 450 | case Inst::Arithmetic: { |
| 451 | Variable *Dest = Instr->getDest(); |
| 452 | const Type DestTy = Dest->getType(); |
| 453 | const InstArithmetic::OpKind Op = |
| 454 | llvm::cast<InstArithmetic>(Instr)->getOp(); |
Eric Holk | cfc2553 | 2016-02-09 17:47:58 -0800 | [diff] [blame] | 455 | if (isVectorType(DestTy)) { |
| 456 | switch (Op) { |
| 457 | default: |
| 458 | break; |
| 459 | case InstArithmetic::Fdiv: |
Eric Holk | 916e37b | 2016-02-17 13:03:29 -0800 | [diff] [blame] | 460 | case InstArithmetic::Frem: |
Eric Holk | cfc2553 | 2016-02-09 17:47:58 -0800 | [diff] [blame] | 461 | case InstArithmetic::Sdiv: |
Eric Holk | 916e37b | 2016-02-17 13:03:29 -0800 | [diff] [blame] | 462 | case InstArithmetic::Srem: |
| 463 | case InstArithmetic::Udiv: |
| 464 | case InstArithmetic::Urem: |
Eric Holk | cfc2553 | 2016-02-09 17:47:58 -0800 | [diff] [blame] | 465 | scalarizeArithmetic(Op, Dest, Instr->getSrc(0), Instr->getSrc(1)); |
| 466 | Instr->setDeleted(); |
| 467 | return; |
| 468 | } |
| 469 | } |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 470 | switch (DestTy) { |
| 471 | default: |
| 472 | return; |
| 473 | case IceType_i64: { |
| 474 | // Technically, ARM has its own aeabi routines, but we can use the |
| 475 | // non-aeabi routine as well. LLVM uses __aeabi_ldivmod for div, but uses |
| 476 | // the more standard __moddi3 for rem. |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 477 | RuntimeHelper HelperID = RuntimeHelper::H_Num; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 478 | switch (Op) { |
| 479 | default: |
| 480 | return; |
| 481 | case InstArithmetic::Udiv: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 482 | HelperID = RuntimeHelper::H_udiv_i64; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 483 | break; |
| 484 | case InstArithmetic::Sdiv: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 485 | HelperID = RuntimeHelper::H_sdiv_i64; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 486 | break; |
| 487 | case InstArithmetic::Urem: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 488 | HelperID = RuntimeHelper::H_urem_i64; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 489 | break; |
| 490 | case InstArithmetic::Srem: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 491 | HelperID = RuntimeHelper::H_srem_i64; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 492 | break; |
| 493 | } |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 494 | Operand *TargetHelper = Ctx->getRuntimeHelperFunc(HelperID); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 495 | ARM32HelpersPreamble[TargetHelper] = &TargetARM32::preambleDivRem; |
| 496 | constexpr SizeT MaxArgs = 2; |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 497 | auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, |
| 498 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 499 | Call->addArg(Instr->getSrc(0)); |
| 500 | Call->addArg(Instr->getSrc(1)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 501 | Instr->setDeleted(); |
| 502 | return; |
| 503 | } |
| 504 | case IceType_i32: |
| 505 | case IceType_i16: |
| 506 | case IceType_i8: { |
| 507 | const bool HasHWDiv = hasCPUFeature(TargetARM32Features::HWDivArm); |
| 508 | InstCast::OpKind CastKind; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 509 | RuntimeHelper HelperID = RuntimeHelper::H_Num; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 510 | switch (Op) { |
| 511 | default: |
| 512 | return; |
| 513 | case InstArithmetic::Udiv: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 514 | HelperID = HasHWDiv ? RuntimeHelper::H_Num : RuntimeHelper::H_udiv_i32; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 515 | CastKind = InstCast::Zext; |
| 516 | break; |
| 517 | case InstArithmetic::Sdiv: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 518 | HelperID = HasHWDiv ? RuntimeHelper::H_Num : RuntimeHelper::H_sdiv_i32; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 519 | CastKind = InstCast::Sext; |
| 520 | break; |
| 521 | case InstArithmetic::Urem: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 522 | HelperID = HasHWDiv ? RuntimeHelper::H_Num : RuntimeHelper::H_urem_i32; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 523 | CastKind = InstCast::Zext; |
| 524 | break; |
| 525 | case InstArithmetic::Srem: |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 526 | HelperID = HasHWDiv ? RuntimeHelper::H_Num : RuntimeHelper::H_srem_i32; |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 527 | CastKind = InstCast::Sext; |
| 528 | break; |
| 529 | } |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 530 | if (HelperID == RuntimeHelper::H_Num) { |
| 531 | // HelperID should only ever be undefined when the processor does not |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 532 | // have a hardware divider. If any other helpers are ever introduced, |
| 533 | // the following assert will have to be modified. |
| 534 | assert(HasHWDiv); |
| 535 | return; |
| 536 | } |
| 537 | Operand *Src0 = Instr->getSrc(0); |
| 538 | Operand *Src1 = Instr->getSrc(1); |
| 539 | if (DestTy != IceType_i32) { |
| 540 | // Src0 and Src1 have to be zero-, or signed-extended to i32. For Src0, |
| 541 | // we just insert a InstCast right before the call to the helper. |
| 542 | Variable *Src0_32 = Func->makeVariable(IceType_i32); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 543 | Context.insert<InstCast>(CastKind, Src0_32, Src0); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 544 | Src0 = Src0_32; |
| 545 | |
| 546 | // For extending Src1, we will just insert an InstCast if Src1 is not a |
| 547 | // Constant. If it is, then we extend it here, and not during program |
| 548 | // runtime. This allows preambleDivRem to optimize-out the div-by-0 |
| 549 | // check. |
| 550 | if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { |
| 551 | const int32_t ShAmt = (DestTy == IceType_i16) ? 16 : 24; |
| 552 | int32_t NewC = C->getValue(); |
| 553 | if (CastKind == InstCast::Zext) { |
| 554 | NewC &= ~(0x80000000l >> ShAmt); |
| 555 | } else { |
| 556 | NewC = (NewC << ShAmt) >> ShAmt; |
| 557 | } |
| 558 | Src1 = Ctx->getConstantInt32(NewC); |
| 559 | } else { |
| 560 | Variable *Src1_32 = Func->makeVariable(IceType_i32); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 561 | Context.insert<InstCast>(CastKind, Src1_32, Src1); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 562 | Src1 = Src1_32; |
| 563 | } |
| 564 | } |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 565 | Operand *TargetHelper = Ctx->getRuntimeHelperFunc(HelperID); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 566 | ARM32HelpersPreamble[TargetHelper] = &TargetARM32::preambleDivRem; |
| 567 | constexpr SizeT MaxArgs = 2; |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 568 | auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, |
| 569 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 570 | assert(Src0->getType() == IceType_i32); |
| 571 | Call->addArg(Src0); |
| 572 | assert(Src1->getType() == IceType_i32); |
| 573 | Call->addArg(Src1); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 574 | Instr->setDeleted(); |
| 575 | return; |
| 576 | } |
| 577 | case IceType_f64: |
| 578 | case IceType_f32: { |
| 579 | if (Op != InstArithmetic::Frem) { |
| 580 | return; |
| 581 | } |
| 582 | constexpr SizeT MaxArgs = 2; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 583 | Operand *TargetHelper = Ctx->getRuntimeHelperFunc( |
| 584 | DestTy == IceType_f32 ? RuntimeHelper::H_frem_f32 |
| 585 | : RuntimeHelper::H_frem_f64); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 586 | auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, |
| 587 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 588 | Call->addArg(Instr->getSrc(0)); |
| 589 | Call->addArg(Instr->getSrc(1)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 590 | Instr->setDeleted(); |
| 591 | return; |
| 592 | } |
| 593 | } |
| 594 | llvm::report_fatal_error("Control flow should never have reached here."); |
| 595 | } |
| 596 | case Inst::Cast: { |
| 597 | Variable *Dest = Instr->getDest(); |
| 598 | Operand *Src0 = Instr->getSrc(0); |
| 599 | const Type DestTy = Dest->getType(); |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 600 | const Type SrcTy = Src0->getType(); |
Eric Holk | cc69fa2 | 2016-02-10 13:07:06 -0800 | [diff] [blame] | 601 | auto *CastInstr = llvm::cast<InstCast>(Instr); |
| 602 | const InstCast::OpKind CastKind = CastInstr->getCastKind(); |
| 603 | |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 604 | switch (CastKind) { |
| 605 | default: |
| 606 | return; |
| 607 | case InstCast::Fptosi: |
| 608 | case InstCast::Fptoui: { |
| 609 | if (DestTy != IceType_i64) { |
| 610 | return; |
| 611 | } |
| 612 | const bool DestIsSigned = CastKind == InstCast::Fptosi; |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 613 | const bool Src0IsF32 = isFloat32Asserting32Or64(SrcTy); |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 614 | Operand *TargetHelper = Ctx->getRuntimeHelperFunc( |
| 615 | Src0IsF32 ? (DestIsSigned ? RuntimeHelper::H_fptosi_f32_i64 |
| 616 | : RuntimeHelper::H_fptoui_f32_i64) |
| 617 | : (DestIsSigned ? RuntimeHelper::H_fptosi_f64_i64 |
| 618 | : RuntimeHelper::H_fptoui_f64_i64)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 619 | static constexpr SizeT MaxArgs = 1; |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 620 | auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, |
| 621 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 622 | Call->addArg(Src0); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 623 | Instr->setDeleted(); |
| 624 | return; |
| 625 | } |
| 626 | case InstCast::Sitofp: |
| 627 | case InstCast::Uitofp: { |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 628 | if (SrcTy != IceType_i64) { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 629 | return; |
| 630 | } |
| 631 | const bool SourceIsSigned = CastKind == InstCast::Sitofp; |
| 632 | const bool DestIsF32 = isFloat32Asserting32Or64(Dest->getType()); |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 633 | Operand *TargetHelper = Ctx->getRuntimeHelperFunc( |
| 634 | DestIsF32 ? (SourceIsSigned ? RuntimeHelper::H_sitofp_i64_f32 |
| 635 | : RuntimeHelper::H_uitofp_i64_f32) |
| 636 | : (SourceIsSigned ? RuntimeHelper::H_sitofp_i64_f64 |
| 637 | : RuntimeHelper::H_uitofp_i64_f64)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 638 | static constexpr SizeT MaxArgs = 1; |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 639 | auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, |
| 640 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 641 | Call->addArg(Src0); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 642 | Instr->setDeleted(); |
| 643 | return; |
| 644 | } |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 645 | case InstCast::Bitcast: { |
| 646 | if (DestTy == SrcTy) { |
| 647 | return; |
| 648 | } |
| 649 | Variable *CallDest = Dest; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 650 | RuntimeHelper HelperID = RuntimeHelper::H_Num; |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 651 | switch (DestTy) { |
| 652 | default: |
| 653 | return; |
| 654 | case IceType_i8: |
| 655 | assert(SrcTy == IceType_v8i1); |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 656 | HelperID = RuntimeHelper::H_bitcast_8xi1_i8; |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 657 | CallDest = Func->makeVariable(IceType_i32); |
| 658 | break; |
| 659 | case IceType_i16: |
| 660 | assert(SrcTy == IceType_v16i1); |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 661 | HelperID = RuntimeHelper::H_bitcast_16xi1_i16; |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 662 | CallDest = Func->makeVariable(IceType_i32); |
| 663 | break; |
| 664 | case IceType_v8i1: { |
| 665 | assert(SrcTy == IceType_i8); |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 666 | HelperID = RuntimeHelper::H_bitcast_i8_8xi1; |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 667 | Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); |
| 668 | // Arguments to functions are required to be at least 32 bits wide. |
| 669 | Context.insert<InstCast>(InstCast::Zext, Src0AsI32, Src0); |
| 670 | Src0 = Src0AsI32; |
| 671 | } break; |
| 672 | case IceType_v16i1: { |
| 673 | assert(SrcTy == IceType_i16); |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 674 | HelperID = RuntimeHelper::H_bitcast_i16_16xi1; |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 675 | Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); |
| 676 | // Arguments to functions are required to be at least 32 bits wide. |
| 677 | Context.insert<InstCast>(InstCast::Zext, Src0AsI32, Src0); |
| 678 | Src0 = Src0AsI32; |
| 679 | } break; |
| 680 | } |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 681 | constexpr SizeT MaxSrcs = 1; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 682 | InstCall *Call = makeHelperCall(HelperID, CallDest, MaxSrcs); |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 683 | Call->addArg(Src0); |
| 684 | Context.insert(Call); |
| 685 | // The PNaCl ABI disallows i8/i16 return types, so truncate the helper |
| 686 | // call result to the appropriate type as necessary. |
| 687 | if (CallDest->getType() != Dest->getType()) |
| 688 | Context.insert<InstCast>(InstCast::Trunc, Dest, CallDest); |
| 689 | Instr->setDeleted(); |
| 690 | return; |
| 691 | } |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 692 | case InstCast::Trunc: { |
| 693 | if (DestTy == SrcTy) { |
| 694 | return; |
| 695 | } |
| 696 | if (!isVectorType(SrcTy)) { |
| 697 | return; |
| 698 | } |
| 699 | assert(typeNumElements(DestTy) == typeNumElements(SrcTy)); |
| 700 | assert(typeElementType(DestTy) == IceType_i1); |
| 701 | assert(isVectorIntegerType(SrcTy)); |
| 702 | return; |
| 703 | } |
| 704 | case InstCast::Sext: |
| 705 | case InstCast::Zext: { |
| 706 | if (DestTy == SrcTy) { |
| 707 | return; |
| 708 | } |
| 709 | if (!isVectorType(DestTy)) { |
| 710 | return; |
| 711 | } |
| 712 | assert(typeNumElements(DestTy) == typeNumElements(SrcTy)); |
| 713 | assert(typeElementType(SrcTy) == IceType_i1); |
| 714 | assert(isVectorIntegerType(DestTy)); |
| 715 | return; |
| 716 | } |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 717 | } |
| 718 | llvm::report_fatal_error("Control flow should never have reached here."); |
| 719 | } |
| 720 | case Inst::IntrinsicCall: { |
| 721 | Variable *Dest = Instr->getDest(); |
| 722 | auto *IntrinsicCall = llvm::cast<InstIntrinsicCall>(Instr); |
| 723 | Intrinsics::IntrinsicID ID = IntrinsicCall->getIntrinsicInfo().ID; |
| 724 | switch (ID) { |
| 725 | default: |
| 726 | return; |
| 727 | case Intrinsics::Ctpop: { |
| 728 | Operand *Src0 = IntrinsicCall->getArg(0); |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 729 | Operand *TargetHelper = |
| 730 | Ctx->getRuntimeHelperFunc(isInt32Asserting32Or64(Src0->getType()) |
| 731 | ? RuntimeHelper::H_call_ctpop_i32 |
| 732 | : RuntimeHelper::H_call_ctpop_i64); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 733 | static constexpr SizeT MaxArgs = 1; |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 734 | auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, |
| 735 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 736 | Call->addArg(Src0); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 737 | Instr->setDeleted(); |
| 738 | if (Src0->getType() == IceType_i64) { |
| 739 | ARM32HelpersPostamble[TargetHelper] = &TargetARM32::postambleCtpop64; |
| 740 | } |
| 741 | return; |
| 742 | } |
| 743 | case Intrinsics::Longjmp: { |
| 744 | static constexpr SizeT MaxArgs = 2; |
| 745 | static constexpr Variable *NoDest = nullptr; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 746 | Operand *TargetHelper = |
| 747 | Ctx->getRuntimeHelperFunc(RuntimeHelper::H_call_longjmp); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 748 | auto *Call = Context.insert<InstCall>(MaxArgs, NoDest, TargetHelper, |
| 749 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 750 | Call->addArg(IntrinsicCall->getArg(0)); |
| 751 | Call->addArg(IntrinsicCall->getArg(1)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 752 | Instr->setDeleted(); |
| 753 | return; |
| 754 | } |
| 755 | case Intrinsics::Memcpy: { |
| 756 | // In the future, we could potentially emit an inline memcpy/memset, etc. |
| 757 | // for intrinsic calls w/ a known length. |
| 758 | static constexpr SizeT MaxArgs = 3; |
| 759 | static constexpr Variable *NoDest = nullptr; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 760 | Operand *TargetHelper = |
| 761 | Ctx->getRuntimeHelperFunc(RuntimeHelper::H_call_memcpy); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 762 | auto *Call = Context.insert<InstCall>(MaxArgs, NoDest, TargetHelper, |
| 763 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 764 | Call->addArg(IntrinsicCall->getArg(0)); |
| 765 | Call->addArg(IntrinsicCall->getArg(1)); |
| 766 | Call->addArg(IntrinsicCall->getArg(2)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 767 | Instr->setDeleted(); |
| 768 | return; |
| 769 | } |
| 770 | case Intrinsics::Memmove: { |
| 771 | static constexpr SizeT MaxArgs = 3; |
| 772 | static constexpr Variable *NoDest = nullptr; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 773 | Operand *TargetHelper = |
| 774 | Ctx->getRuntimeHelperFunc(RuntimeHelper::H_call_memmove); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 775 | auto *Call = Context.insert<InstCall>(MaxArgs, NoDest, TargetHelper, |
| 776 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 777 | Call->addArg(IntrinsicCall->getArg(0)); |
| 778 | Call->addArg(IntrinsicCall->getArg(1)); |
| 779 | Call->addArg(IntrinsicCall->getArg(2)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 780 | Instr->setDeleted(); |
| 781 | return; |
| 782 | } |
| 783 | case Intrinsics::Memset: { |
| 784 | // The value operand needs to be extended to a stack slot size because the |
| 785 | // PNaCl ABI requires arguments to be at least 32 bits wide. |
| 786 | Operand *ValOp = IntrinsicCall->getArg(1); |
| 787 | assert(ValOp->getType() == IceType_i8); |
| 788 | Variable *ValExt = Func->makeVariable(stackSlotType()); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 789 | Context.insert<InstCast>(InstCast::Zext, ValExt, ValOp); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 790 | |
| 791 | // Technically, ARM has its own __aeabi_memset, but we can use plain |
| 792 | // memset too. The value and size argument need to be flipped if we ever |
| 793 | // decide to use __aeabi_memset. |
| 794 | static constexpr SizeT MaxArgs = 3; |
| 795 | static constexpr Variable *NoDest = nullptr; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 796 | Operand *TargetHelper = |
| 797 | Ctx->getRuntimeHelperFunc(RuntimeHelper::H_call_memset); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 798 | auto *Call = Context.insert<InstCall>(MaxArgs, NoDest, TargetHelper, |
| 799 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 800 | Call->addArg(IntrinsicCall->getArg(0)); |
| 801 | Call->addArg(ValExt); |
| 802 | Call->addArg(IntrinsicCall->getArg(2)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 803 | Instr->setDeleted(); |
| 804 | return; |
| 805 | } |
| 806 | case Intrinsics::NaClReadTP: { |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 807 | if (SandboxingType == ST_NaCl) { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 808 | return; |
| 809 | } |
| 810 | static constexpr SizeT MaxArgs = 0; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 811 | Operand *TargetHelper = |
| 812 | SandboxingType == ST_Nonsfi |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 813 | ? Ctx->getConstantExternSym( |
| 814 | Ctx->getGlobalString("__aeabi_read_tp")) |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 815 | : Ctx->getRuntimeHelperFunc(RuntimeHelper::H_call_read_tp); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 816 | Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, NoTailCall, |
| 817 | IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 818 | Instr->setDeleted(); |
| 819 | return; |
| 820 | } |
| 821 | case Intrinsics::Setjmp: { |
| 822 | static constexpr SizeT MaxArgs = 1; |
Karl Schimpf | 20070e8 | 2016-03-17 13:30:13 -0700 | [diff] [blame] | 823 | Operand *TargetHelper = |
| 824 | Ctx->getRuntimeHelperFunc(RuntimeHelper::H_call_setjmp); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 825 | auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper, |
| 826 | NoTailCall, IsTargetHelperCall); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 827 | Call->addArg(IntrinsicCall->getArg(0)); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 828 | Instr->setDeleted(); |
| 829 | return; |
| 830 | } |
| 831 | } |
| 832 | llvm::report_fatal_error("Control flow should never have reached here."); |
| 833 | } |
| 834 | } |
| 835 | } |
| 836 | |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 837 | void TargetARM32::findMaxStackOutArgsSize() { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 838 | // MinNeededOutArgsBytes should be updated if the Target ever creates a |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 839 | // high-level InstCall that requires more stack bytes. |
| 840 | constexpr size_t MinNeededOutArgsBytes = 0; |
| 841 | MaxOutArgsSizeBytes = MinNeededOutArgsBytes; |
| 842 | for (CfgNode *Node : Func->getNodes()) { |
| 843 | Context.init(Node); |
| 844 | while (!Context.atEnd()) { |
| 845 | PostIncrLoweringContext PostIncrement(Context); |
Jim Stichnoth | f5fdd23 | 2016-05-09 12:24:36 -0700 | [diff] [blame] | 846 | Inst *CurInstr = iteratorToInst(Context.getCur()); |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 847 | if (auto *Call = llvm::dyn_cast<InstCall>(CurInstr)) { |
| 848 | SizeT OutArgsSizeBytes = getCallStackArgumentsSizeBytes(Call); |
| 849 | MaxOutArgsSizeBytes = std::max(MaxOutArgsSizeBytes, OutArgsSizeBytes); |
| 850 | } |
| 851 | } |
| 852 | } |
| 853 | } |
| 854 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 855 | void TargetARM32::createGotPtr() { |
| 856 | if (SandboxingType != ST_Nonsfi) { |
| 857 | return; |
| 858 | } |
| 859 | GotPtr = Func->makeVariable(IceType_i32); |
| 860 | } |
| 861 | |
| 862 | void TargetARM32::insertGotPtrInitPlaceholder() { |
| 863 | if (SandboxingType != ST_Nonsfi) { |
| 864 | return; |
| 865 | } |
| 866 | assert(GotPtr != nullptr); |
| 867 | // We add the two placeholder instructions here. The first fakedefs T, an |
| 868 | // infinite-weight temporary, while the second fakedefs the GotPtr "using" T. |
| 869 | // This is needed because the GotPtr initialization, if needed, will require |
| 870 | // a register: |
| 871 | // |
| 872 | // movw reg, _GLOBAL_OFFSET_TABLE_ - 16 - . |
| 873 | // movt reg, _GLOBAL_OFFSET_TABLE_ - 12 - . |
| 874 | // add reg, pc, reg |
| 875 | // mov GotPtr, reg |
| 876 | // |
| 877 | // If GotPtr is not used, then both these pseudo-instructions are dce'd. |
| 878 | Variable *T = makeReg(IceType_i32); |
| 879 | Context.insert<InstFakeDef>(T); |
| 880 | Context.insert<InstFakeDef>(GotPtr, T); |
| 881 | } |
| 882 | |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 883 | GlobalString |
| 884 | TargetARM32::createGotoffRelocation(const ConstantRelocatable *CR) { |
| 885 | GlobalString CRName = CR->getName(); |
| 886 | GlobalString CRGotoffName = |
| 887 | Ctx->getGlobalString("GOTOFF$" + Func->getFunctionName() + "$" + CRName); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 888 | if (KnownGotoffs.count(CRGotoffName) == 0) { |
Jim Stichnoth | 98ba006 | 2016-03-07 09:26:22 -0800 | [diff] [blame] | 889 | constexpr bool SuppressMangling = true; |
John Porto | a78e4ba | 2016-03-15 09:28:04 -0700 | [diff] [blame] | 890 | auto *Global = |
| 891 | VariableDeclaration::create(Func->getGlobalPool(), SuppressMangling); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 892 | Global->setIsConstant(true); |
| 893 | Global->setName(CRName); |
John Porto | a78e4ba | 2016-03-15 09:28:04 -0700 | [diff] [blame] | 894 | Func->getGlobalPool()->willNotBeEmitted(Global); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 895 | |
John Porto | a78e4ba | 2016-03-15 09:28:04 -0700 | [diff] [blame] | 896 | auto *Gotoff = |
| 897 | VariableDeclaration::create(Func->getGlobalPool(), SuppressMangling); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 898 | constexpr auto GotFixup = R_ARM_GOTOFF32; |
| 899 | Gotoff->setIsConstant(true); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 900 | Gotoff->addInitializer(VariableDeclaration::RelocInitializer::create( |
John Porto | a78e4ba | 2016-03-15 09:28:04 -0700 | [diff] [blame] | 901 | Func->getGlobalPool(), Global, {RelocOffset::create(Ctx, 0)}, |
| 902 | GotFixup)); |
Jim Stichnoth | 98ba006 | 2016-03-07 09:26:22 -0800 | [diff] [blame] | 903 | Gotoff->setName(CRGotoffName); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 904 | Func->addGlobal(Gotoff); |
| 905 | KnownGotoffs.emplace(CRGotoffName); |
| 906 | } |
| 907 | return CRGotoffName; |
| 908 | } |
| 909 | |
| 910 | void TargetARM32::materializeGotAddr(CfgNode *Node) { |
| 911 | if (SandboxingType != ST_Nonsfi) { |
| 912 | return; |
| 913 | } |
| 914 | |
| 915 | // At first, we try to find the |
| 916 | // GotPtr = def T |
| 917 | // pseudo-instruction that we placed for defining the got ptr. That |
| 918 | // instruction is not just a place-holder for defining the GotPtr (thus |
| 919 | // keeping liveness consistent), but it is also located at a point where it is |
| 920 | // safe to materialize the got addr -- i.e., before loading parameters to |
| 921 | // registers, but after moving register parameters from their home location. |
| 922 | InstFakeDef *DefGotPtr = nullptr; |
| 923 | for (auto &Inst : Node->getInsts()) { |
| 924 | auto *FakeDef = llvm::dyn_cast<InstFakeDef>(&Inst); |
| 925 | if (FakeDef != nullptr && FakeDef->getDest() == GotPtr) { |
| 926 | DefGotPtr = FakeDef; |
| 927 | break; |
| 928 | } |
| 929 | } |
| 930 | |
| 931 | if (DefGotPtr == nullptr || DefGotPtr->isDeleted()) { |
| 932 | return; |
| 933 | } |
| 934 | |
| 935 | // The got addr needs to be materialized at the same point where DefGotPtr |
| 936 | // lives. |
Jim Stichnoth | f5fdd23 | 2016-05-09 12:24:36 -0700 | [diff] [blame] | 937 | Context.setInsertPoint(instToIterator(DefGotPtr)); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 938 | assert(DefGotPtr->getSrcSize() == 1); |
| 939 | auto *T = llvm::cast<Variable>(DefGotPtr->getSrc(0)); |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 940 | loadNamedConstantRelocatablePIC(Ctx->getGlobalString(GlobalOffsetTable), T, |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 941 | [this, T](Variable *PC) { _add(T, PC, T); }); |
| 942 | _mov(GotPtr, T); |
| 943 | DefGotPtr->setDeleted(); |
| 944 | } |
| 945 | |
| 946 | void TargetARM32::loadNamedConstantRelocatablePIC( |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 947 | GlobalString Name, Variable *Register, |
Jim Stichnoth | 98ba006 | 2016-03-07 09:26:22 -0800 | [diff] [blame] | 948 | std::function<void(Variable *PC)> Finish) { |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 949 | assert(SandboxingType == ST_Nonsfi); |
| 950 | // We makeReg() here instead of getPhysicalRegister() because the latter ends |
| 951 | // up creating multi-blocks temporaries that liveness fails to validate. |
| 952 | auto *PC = makeReg(IceType_i32, RegARM32::Reg_pc); |
| 953 | |
| 954 | auto *AddPcReloc = RelocOffset::create(Ctx); |
| 955 | AddPcReloc->setSubtract(true); |
| 956 | auto *AddPcLabel = InstARM32Label::create(Func, this); |
| 957 | AddPcLabel->setRelocOffset(AddPcReloc); |
| 958 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 959 | auto *MovwReloc = RelocOffset::create(Ctx); |
| 960 | auto *MovwLabel = InstARM32Label::create(Func, this); |
| 961 | MovwLabel->setRelocOffset(MovwReloc); |
| 962 | |
| 963 | auto *MovtReloc = RelocOffset::create(Ctx); |
| 964 | auto *MovtLabel = InstARM32Label::create(Func, this); |
| 965 | MovtLabel->setRelocOffset(MovtReloc); |
| 966 | |
| 967 | // The EmitString for these constant relocatables have hardcoded offsets |
| 968 | // attached to them. This could be dangerous if, e.g., we ever implemented |
| 969 | // instruction scheduling but llvm-mc currently does not support |
| 970 | // |
| 971 | // movw reg, #:lower16:(Symbol - Label - Number) |
| 972 | // movt reg, #:upper16:(Symbol - Label - Number) |
| 973 | // |
| 974 | // relocations. |
John Porto | e82b560 | 2016-02-24 15:58:55 -0800 | [diff] [blame] | 975 | static constexpr RelocOffsetT PcOffset = -8; |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 976 | auto *CRLower = Ctx->getConstantSymWithEmitString( |
| 977 | PcOffset, {MovwReloc, AddPcReloc}, Name, Name + " -16"); |
| 978 | auto *CRUpper = Ctx->getConstantSymWithEmitString( |
| 979 | PcOffset, {MovtReloc, AddPcReloc}, Name, Name + " -12"); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 980 | |
| 981 | Context.insert(MovwLabel); |
| 982 | _movw(Register, CRLower); |
| 983 | Context.insert(MovtLabel); |
| 984 | _movt(Register, CRUpper); |
| 985 | // PC = fake-def to keep liveness consistent. |
| 986 | Context.insert<InstFakeDef>(PC); |
| 987 | Context.insert(AddPcLabel); |
| 988 | Finish(PC); |
| 989 | } |
| 990 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 991 | void TargetARM32::translateO2() { |
| 992 | TimerMarker T(TimerStack::TT_O2, Func); |
| 993 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 994 | // TODO(stichnot): share passes with other targets? |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 995 | // https://code.google.com/p/nativeclient/issues/detail?id=4094 |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 996 | if (SandboxingType == ST_Nonsfi) { |
| 997 | createGotPtr(); |
| 998 | } |
John Porto | 5e0a8a7 | 2015-11-20 13:50:36 -0800 | [diff] [blame] | 999 | genTargetHelperCalls(); |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1000 | findMaxStackOutArgsSize(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1001 | |
David Sehr | 4318a41 | 2015-11-11 15:01:55 -0800 | [diff] [blame] | 1002 | // Do not merge Alloca instructions, and lay out the stack. |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1003 | static constexpr bool SortAndCombineAllocas = true; |
David Sehr | 4318a41 | 2015-11-11 15:01:55 -0800 | [diff] [blame] | 1004 | Func->processAllocas(SortAndCombineAllocas); |
| 1005 | Func->dump("After Alloca processing"); |
| 1006 | |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 1007 | if (!getFlags().getEnablePhiEdgeSplit()) { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1008 | // Lower Phi instructions. |
| 1009 | Func->placePhiLoads(); |
| 1010 | if (Func->hasError()) |
| 1011 | return; |
| 1012 | Func->placePhiStores(); |
| 1013 | if (Func->hasError()) |
| 1014 | return; |
| 1015 | Func->deletePhis(); |
| 1016 | if (Func->hasError()) |
| 1017 | return; |
| 1018 | Func->dump("After Phi lowering"); |
| 1019 | } |
| 1020 | |
| 1021 | // Address mode optimization. |
| 1022 | Func->getVMetadata()->init(VMK_SingleDefs); |
| 1023 | Func->doAddressOpt(); |
John Porto | a47c11c | 2016-04-21 05:53:42 -0700 | [diff] [blame] | 1024 | Func->materializeVectorShuffles(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1025 | |
| 1026 | // Argument lowering |
| 1027 | Func->doArgLowering(); |
| 1028 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1029 | // Target lowering. This requires liveness analysis for some parts of the |
| 1030 | // lowering decisions, such as compare/branch fusing. If non-lightweight |
| 1031 | // liveness analysis is used, the instructions need to be renumbered first. |
| 1032 | // TODO: This renumbering should only be necessary if we're actually |
| 1033 | // calculating live intervals, which we only do for register allocation. |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1034 | Func->renumberInstructions(); |
| 1035 | if (Func->hasError()) |
| 1036 | return; |
| 1037 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1038 | // TODO: It should be sufficient to use the fastest liveness calculation, |
| 1039 | // i.e. livenessLightweight(). However, for some reason that slows down the |
| 1040 | // rest of the translation. Investigate. |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1041 | Func->liveness(Liveness_Basic); |
| 1042 | if (Func->hasError()) |
| 1043 | return; |
| 1044 | Func->dump("After ARM32 address mode opt"); |
| 1045 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 1046 | if (SandboxingType == ST_Nonsfi) { |
| 1047 | insertGotPtrInitPlaceholder(); |
| 1048 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1049 | Func->genCode(); |
| 1050 | if (Func->hasError()) |
| 1051 | return; |
| 1052 | Func->dump("After ARM32 codegen"); |
| 1053 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1054 | // Register allocation. This requires instruction renumbering and full |
| 1055 | // liveness analysis. |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1056 | Func->renumberInstructions(); |
| 1057 | if (Func->hasError()) |
| 1058 | return; |
| 1059 | Func->liveness(Liveness_Intervals); |
| 1060 | if (Func->hasError()) |
| 1061 | return; |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1062 | // The post-codegen dump is done here, after liveness analysis and associated |
| 1063 | // cleanup, to make the dump cleaner and more useful. |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1064 | Func->dump("After initial ARM32 codegen"); |
Jim Stichnoth | 2943d77 | 2016-06-21 11:22:17 -0700 | [diff] [blame] | 1065 | // Validate the live range computations. The expensive validation call is |
| 1066 | // deliberately only made when assertions are enabled. |
| 1067 | assert(Func->validateLiveness()); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1068 | Func->getVMetadata()->init(VMK_All); |
| 1069 | regAlloc(RAK_Global); |
| 1070 | if (Func->hasError()) |
| 1071 | return; |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1072 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 1073 | copyRegAllocFromInfWeightVariable64On32(Func->getVariables()); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1074 | Func->dump("After linear scan regalloc"); |
| 1075 | |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 1076 | if (getFlags().getEnablePhiEdgeSplit()) { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1077 | Func->advancedPhiLowering(); |
| 1078 | Func->dump("After advanced Phi lowering"); |
| 1079 | } |
| 1080 | |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1081 | ForbidTemporaryWithoutReg _(this); |
| 1082 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1083 | // Stack frame mapping. |
| 1084 | Func->genFrame(); |
| 1085 | if (Func->hasError()) |
| 1086 | return; |
| 1087 | Func->dump("After stack frame mapping"); |
| 1088 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1089 | postLowerLegalization(); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1090 | if (Func->hasError()) |
| 1091 | return; |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1092 | Func->dump("After postLowerLegalization"); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1093 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1094 | Func->contractEmptyNodes(); |
| 1095 | Func->reorderNodes(); |
| 1096 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1097 | // Branch optimization. This needs to be done just before code emission. In |
| 1098 | // particular, no transformations that insert or reorder CfgNodes should be |
| 1099 | // done after branch optimization. We go ahead and do it before nop insertion |
| 1100 | // to reduce the amount of work needed for searching for opportunities. |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1101 | Func->doBranchOpt(); |
| 1102 | Func->dump("After branch optimization"); |
| 1103 | |
| 1104 | // Nop insertion |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 1105 | if (getFlags().getShouldDoNopInsertion()) { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1106 | Func->doNopInsertion(); |
| 1107 | } |
| 1108 | } |
| 1109 | |
| 1110 | void TargetARM32::translateOm1() { |
| 1111 | TimerMarker T(TimerStack::TT_Om1, Func); |
| 1112 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 1113 | // TODO(stichnot): share passes with other targets? |
| 1114 | if (SandboxingType == ST_Nonsfi) { |
| 1115 | createGotPtr(); |
| 1116 | } |
| 1117 | |
John Porto | 5e0a8a7 | 2015-11-20 13:50:36 -0800 | [diff] [blame] | 1118 | genTargetHelperCalls(); |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1119 | findMaxStackOutArgsSize(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1120 | |
David Sehr | 4318a41 | 2015-11-11 15:01:55 -0800 | [diff] [blame] | 1121 | // Do not merge Alloca instructions, and lay out the stack. |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1122 | static constexpr bool DontSortAndCombineAllocas = false; |
| 1123 | Func->processAllocas(DontSortAndCombineAllocas); |
David Sehr | 4318a41 | 2015-11-11 15:01:55 -0800 | [diff] [blame] | 1124 | Func->dump("After Alloca processing"); |
| 1125 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1126 | Func->placePhiLoads(); |
| 1127 | if (Func->hasError()) |
| 1128 | return; |
| 1129 | Func->placePhiStores(); |
| 1130 | if (Func->hasError()) |
| 1131 | return; |
| 1132 | Func->deletePhis(); |
| 1133 | if (Func->hasError()) |
| 1134 | return; |
| 1135 | Func->dump("After Phi lowering"); |
| 1136 | |
| 1137 | Func->doArgLowering(); |
| 1138 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 1139 | if (SandboxingType == ST_Nonsfi) { |
| 1140 | insertGotPtrInitPlaceholder(); |
| 1141 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1142 | Func->genCode(); |
| 1143 | if (Func->hasError()) |
| 1144 | return; |
| 1145 | Func->dump("After initial ARM32 codegen"); |
| 1146 | |
| 1147 | regAlloc(RAK_InfOnly); |
| 1148 | if (Func->hasError()) |
| 1149 | return; |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1150 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 1151 | copyRegAllocFromInfWeightVariable64On32(Func->getVariables()); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1152 | Func->dump("After regalloc of infinite-weight variables"); |
| 1153 | |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1154 | ForbidTemporaryWithoutReg _(this); |
| 1155 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1156 | Func->genFrame(); |
| 1157 | if (Func->hasError()) |
| 1158 | return; |
| 1159 | Func->dump("After stack frame mapping"); |
| 1160 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1161 | postLowerLegalization(); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1162 | if (Func->hasError()) |
| 1163 | return; |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1164 | Func->dump("After postLowerLegalization"); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1165 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1166 | // Nop insertion |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 1167 | if (getFlags().getShouldDoNopInsertion()) { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1168 | Func->doNopInsertion(); |
| 1169 | } |
| 1170 | } |
| 1171 | |
David Sehr | e39d0ca | 2015-11-06 11:25:41 -0800 | [diff] [blame] | 1172 | uint32_t TargetARM32::getStackAlignment() const { |
| 1173 | return ARM32_STACK_ALIGNMENT_BYTES; |
| 1174 | } |
| 1175 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1176 | bool TargetARM32::doBranchOpt(Inst *I, const CfgNode *NextNode) { |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 1177 | if (auto *Br = llvm::dyn_cast<InstARM32Br>(I)) { |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 1178 | return Br->optimizeBranch(NextNode); |
| 1179 | } |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 1180 | return false; |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1181 | } |
| 1182 | |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 1183 | const char *TargetARM32::getRegName(RegNumT RegNum, Type Ty) const { |
Karl Schimpf | 7cb2db3 | 2015-10-29 14:04:12 -0700 | [diff] [blame] | 1184 | (void)Ty; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 1185 | return RegARM32::getRegName(RegNum); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1186 | } |
| 1187 | |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1188 | Variable *TargetARM32::getPhysicalRegister(RegNumT RegNum, Type Ty) { |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 1189 | static const Type DefaultType[] = { |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1190 | #define X(val, encode, name, cc_arg, scratch, preserved, stackptr, frameptr, \ |
John Porto | dff7dbd | 2016-01-04 09:49:55 -0800 | [diff] [blame] | 1191 | isGPR, isInt, isI64Pair, isFP32, isFP64, isVec128, alias_init) \ |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 1192 | (isFP32) \ |
| 1193 | ? IceType_f32 \ |
| 1194 | : ((isFP64) ? IceType_f64 : ((isVec128 ? IceType_v4i32 : IceType_i32))), |
| 1195 | REGARM32_TABLE |
| 1196 | #undef X |
| 1197 | }; |
| 1198 | |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 1199 | if (Ty == IceType_void) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1200 | assert(unsigned(RegNum) < llvm::array_lengthof(DefaultType)); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 1201 | Ty = DefaultType[RegNum]; |
| 1202 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1203 | if (PhysicalRegisters[Ty].empty()) |
| 1204 | PhysicalRegisters[Ty].resize(RegARM32::Reg_NUM); |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1205 | assert(unsigned(RegNum) < PhysicalRegisters[Ty].size()); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1206 | Variable *Reg = PhysicalRegisters[Ty][RegNum]; |
| 1207 | if (Reg == nullptr) { |
| 1208 | Reg = Func->makeVariable(Ty); |
| 1209 | Reg->setRegNum(RegNum); |
| 1210 | PhysicalRegisters[Ty][RegNum] = Reg; |
Jim Stichnoth | 6966055 | 2015-09-18 06:41:02 -0700 | [diff] [blame] | 1211 | // Specially mark a named physical register as an "argument" so that it is |
| 1212 | // considered live upon function entry. Otherwise it's possible to get |
| 1213 | // liveness validation errors for saving callee-save registers. |
| 1214 | Func->addImplicitArg(Reg); |
| 1215 | // Don't bother tracking the live range of a named physical register. |
| 1216 | Reg->setIgnoreLiveness(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1217 | } |
| 1218 | return Reg; |
| 1219 | } |
| 1220 | |
Andrew Scull | 86df4e9 | 2015-07-30 13:54:44 -0700 | [diff] [blame] | 1221 | void TargetARM32::emitJumpTable(const Cfg *Func, |
| 1222 | const InstJumpTable *JumpTable) const { |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 1223 | (void)Func; |
Andrew Scull | 86df4e9 | 2015-07-30 13:54:44 -0700 | [diff] [blame] | 1224 | (void)JumpTable; |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 1225 | UnimplementedError(getFlags()); |
Andrew Scull | 86df4e9 | 2015-07-30 13:54:44 -0700 | [diff] [blame] | 1226 | } |
| 1227 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1228 | void TargetARM32::emitVariable(const Variable *Var) const { |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1229 | if (!BuildDefs::dump()) |
| 1230 | return; |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1231 | Ostream &Str = Ctx->getStrEmit(); |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 1232 | if (Var->hasReg()) { |
| 1233 | Str << getRegName(Var->getRegNum(), Var->getType()); |
| 1234 | return; |
| 1235 | } |
Andrew Scull | 11c9a32 | 2015-08-28 14:24:14 -0700 | [diff] [blame] | 1236 | if (Var->mustHaveReg()) { |
Jim Stichnoth | a91c341 | 2016-04-05 15:31:43 -0700 | [diff] [blame] | 1237 | llvm::report_fatal_error("Infinite-weight Variable (" + Var->getName() + |
Jim Stichnoth | 45bec54 | 2016-02-05 10:26:09 -0800 | [diff] [blame] | 1238 | ") has no register assigned - function " + |
| 1239 | Func->getFunctionName()); |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 1240 | } |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1241 | assert(!Var->isRematerializable()); |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 1242 | int32_t Offset = Var->getStackOffset(); |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1243 | auto BaseRegNum = Var->getBaseRegNum(); |
Reed Kotler | 5fa0a5f | 2016-02-15 20:01:24 -0800 | [diff] [blame] | 1244 | if (BaseRegNum.hasNoValue()) { |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1245 | BaseRegNum = getFrameOrStackReg(); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1246 | } |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 1247 | const Type VarTy = Var->getType(); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 1248 | Str << "[" << getRegName(BaseRegNum, VarTy); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1249 | if (Offset != 0) { |
Jim Stichnoth | 8ff4b28 | 2016-01-04 15:39:06 -0800 | [diff] [blame] | 1250 | Str << ", #" << Offset; |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1251 | } |
| 1252 | Str << "]"; |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1253 | } |
| 1254 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1255 | TargetARM32::CallingConv::CallingConv() |
| 1256 | : GPRegsUsed(RegARM32::Reg_NUM), |
| 1257 | GPRArgs(GPRArgInitializer.rbegin(), GPRArgInitializer.rend()), |
| 1258 | I64Args(I64ArgInitializer.rbegin(), I64ArgInitializer.rend()), |
| 1259 | VFPRegsUsed(RegARM32::Reg_NUM), |
| 1260 | FP32Args(FP32ArgInitializer.rbegin(), FP32ArgInitializer.rend()), |
| 1261 | FP64Args(FP64ArgInitializer.rbegin(), FP64ArgInitializer.rend()), |
| 1262 | Vec128Args(Vec128ArgInitializer.rbegin(), Vec128ArgInitializer.rend()) {} |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 1263 | |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1264 | bool TargetARM32::CallingConv::argInGPR(Type Ty, RegNumT *Reg) { |
| 1265 | CfgVector<RegNumT> *Source; |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 1266 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1267 | switch (Ty) { |
| 1268 | default: { |
| 1269 | assert(isScalarIntegerType(Ty)); |
| 1270 | Source = &GPRArgs; |
| 1271 | } break; |
| 1272 | case IceType_i64: { |
| 1273 | Source = &I64Args; |
| 1274 | } break; |
| 1275 | } |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1276 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1277 | discardUnavailableGPRsAndTheirAliases(Source); |
| 1278 | |
| 1279 | if (Source->empty()) { |
| 1280 | GPRegsUsed.set(); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 1281 | return false; |
John Porto | 385351b | 2015-09-16 16:11:10 -0700 | [diff] [blame] | 1282 | } |
| 1283 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1284 | *Reg = Source->back(); |
| 1285 | // Note that we don't Source->pop_back() here. This is intentional. Notice how |
| 1286 | // we mark all of Reg's aliases as Used. So, for the next argument, |
| 1287 | // Source->back() is marked as unavailable, and it is thus implicitly popped |
| 1288 | // from the stack. |
| 1289 | GPRegsUsed |= RegisterAliases[*Reg]; |
| 1290 | return true; |
| 1291 | } |
| 1292 | |
| 1293 | // GPR are not packed when passing parameters. Thus, a function foo(i32, i64, |
| 1294 | // i32) will have the first argument in r0, the second in r1-r2, and the third |
| 1295 | // on the stack. To model this behavior, whenever we pop a register from Regs, |
| 1296 | // we remove all of its aliases from the pool of available GPRs. This has the |
| 1297 | // effect of computing the "closure" on the GPR registers. |
| 1298 | void TargetARM32::CallingConv::discardUnavailableGPRsAndTheirAliases( |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1299 | CfgVector<RegNumT> *Regs) { |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1300 | while (!Regs->empty() && GPRegsUsed[Regs->back()]) { |
| 1301 | GPRegsUsed |= RegisterAliases[Regs->back()]; |
| 1302 | Regs->pop_back(); |
| 1303 | } |
| 1304 | } |
| 1305 | |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1306 | bool TargetARM32::CallingConv::argInVFP(Type Ty, RegNumT *Reg) { |
| 1307 | CfgVector<RegNumT> *Source; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1308 | |
| 1309 | switch (Ty) { |
| 1310 | default: { |
| 1311 | assert(isVectorType(Ty)); |
| 1312 | Source = &Vec128Args; |
| 1313 | } break; |
| 1314 | case IceType_f32: { |
| 1315 | Source = &FP32Args; |
| 1316 | } break; |
| 1317 | case IceType_f64: { |
| 1318 | Source = &FP64Args; |
| 1319 | } break; |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 1320 | } |
John Porto | 385351b | 2015-09-16 16:11:10 -0700 | [diff] [blame] | 1321 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1322 | discardUnavailableVFPRegs(Source); |
| 1323 | |
| 1324 | if (Source->empty()) { |
| 1325 | VFPRegsUsed.set(); |
| 1326 | return false; |
| 1327 | } |
| 1328 | |
| 1329 | *Reg = Source->back(); |
| 1330 | VFPRegsUsed |= RegisterAliases[*Reg]; |
| 1331 | return true; |
| 1332 | } |
| 1333 | |
| 1334 | // Arguments in VFP registers are not packed, so we don't mark the popped |
| 1335 | // registers' aliases as unavailable. |
| 1336 | void TargetARM32::CallingConv::discardUnavailableVFPRegs( |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1337 | CfgVector<RegNumT> *Regs) { |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1338 | while (!Regs->empty() && VFPRegsUsed[Regs->back()]) { |
| 1339 | Regs->pop_back(); |
| 1340 | } |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 1341 | } |
| 1342 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1343 | void TargetARM32::lowerArguments() { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1344 | VarList &Args = Func->getArgs(); |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 1345 | TargetARM32::CallingConv CC; |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1346 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1347 | // For each register argument, replace Arg in the argument list with the home |
| 1348 | // register. Then generate an instruction in the prolog to copy the home |
| 1349 | // register to the assigned location of Arg. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1350 | Context.init(Func->getEntryNode()); |
| 1351 | Context.setInsertPoint(Context.getCur()); |
| 1352 | |
| 1353 | for (SizeT I = 0, E = Args.size(); I < E; ++I) { |
| 1354 | Variable *Arg = Args[I]; |
| 1355 | Type Ty = Arg->getType(); |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1356 | RegNumT RegNum; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1357 | if (isScalarIntegerType(Ty)) { |
| 1358 | if (!CC.argInGPR(Ty, &RegNum)) { |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1359 | continue; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1360 | } |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1361 | } else { |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1362 | if (!CC.argInVFP(Ty, &RegNum)) { |
| 1363 | continue; |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 1364 | } |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1365 | } |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1366 | |
| 1367 | Variable *RegisterArg = Func->makeVariable(Ty); |
| 1368 | if (BuildDefs::dump()) { |
Jim Stichnoth | a91c341 | 2016-04-05 15:31:43 -0700 | [diff] [blame] | 1369 | RegisterArg->setName(Func, "home_reg:" + Arg->getName()); |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1370 | } |
| 1371 | RegisterArg->setIsArg(); |
| 1372 | Arg->setIsArg(false); |
| 1373 | Args[I] = RegisterArg; |
| 1374 | switch (Ty) { |
| 1375 | default: { RegisterArg->setRegNum(RegNum); } break; |
| 1376 | case IceType_i64: { |
| 1377 | auto *RegisterArg64 = llvm::cast<Variable64On32>(RegisterArg); |
| 1378 | RegisterArg64->initHiLo(Func); |
| 1379 | RegisterArg64->getLo()->setRegNum( |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1380 | RegNumT::fixme(RegARM32::getI64PairFirstGPRNum(RegNum))); |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1381 | RegisterArg64->getHi()->setRegNum( |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1382 | RegNumT::fixme(RegARM32::getI64PairSecondGPRNum(RegNum))); |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1383 | } break; |
| 1384 | } |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 1385 | Context.insert<InstAssign>(Arg, RegisterArg); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 1386 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1387 | } |
| 1388 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1389 | // Helper function for addProlog(). |
| 1390 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1391 | // This assumes Arg is an argument passed on the stack. This sets the frame |
| 1392 | // offset for Arg and updates InArgsSizeBytes according to Arg's width. For an |
| 1393 | // I64 arg that has been split into Lo and Hi components, it calls itself |
| 1394 | // recursively on the components, taking care to handle Lo first because of the |
| 1395 | // little-endian architecture. Lastly, this function generates an instruction |
| 1396 | // to copy Arg into its assigned register if applicable. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1397 | void TargetARM32::finishArgumentLowering(Variable *Arg, Variable *FramePtr, |
| 1398 | size_t BasicFrameOffset, |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1399 | size_t *InArgsSizeBytes) { |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 1400 | const Type Ty = Arg->getType(); |
| 1401 | *InArgsSizeBytes = applyStackAlignmentTy(*InArgsSizeBytes, Ty); |
| 1402 | |
Andrew Scull | 6d47bcd | 2015-09-17 17:10:05 -0700 | [diff] [blame] | 1403 | if (auto *Arg64On32 = llvm::dyn_cast<Variable64On32>(Arg)) { |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 1404 | Variable *const Lo = Arg64On32->getLo(); |
| 1405 | Variable *const Hi = Arg64On32->getHi(); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1406 | finishArgumentLowering(Lo, FramePtr, BasicFrameOffset, InArgsSizeBytes); |
| 1407 | finishArgumentLowering(Hi, FramePtr, BasicFrameOffset, InArgsSizeBytes); |
| 1408 | return; |
| 1409 | } |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1410 | assert(Ty != IceType_i64); |
| 1411 | |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1412 | const int32_t ArgStackOffset = BasicFrameOffset + *InArgsSizeBytes; |
| 1413 | *InArgsSizeBytes += typeWidthInBytesOnStack(Ty); |
| 1414 | |
| 1415 | if (!Arg->hasReg()) { |
| 1416 | Arg->setStackOffset(ArgStackOffset); |
| 1417 | return; |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1418 | } |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1419 | |
| 1420 | // If the argument variable has been assigned a register, we need to copy the |
| 1421 | // value from the stack slot. |
| 1422 | Variable *Parameter = Func->makeVariable(Ty); |
| 1423 | Parameter->setMustNotHaveReg(); |
| 1424 | Parameter->setStackOffset(ArgStackOffset); |
| 1425 | _mov(Arg, Parameter); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1426 | } |
| 1427 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1428 | Type TargetARM32::stackSlotType() { return IceType_i32; } |
| 1429 | |
| 1430 | void TargetARM32::addProlog(CfgNode *Node) { |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1431 | // Stack frame layout: |
| 1432 | // |
| 1433 | // +------------------------+ |
| 1434 | // | 1. preserved registers | |
| 1435 | // +------------------------+ |
| 1436 | // | 2. padding | |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1437 | // +------------------------+ <--- FramePointer (if used) |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1438 | // | 3. global spill area | |
| 1439 | // +------------------------+ |
| 1440 | // | 4. padding | |
| 1441 | // +------------------------+ |
| 1442 | // | 5. local spill area | |
| 1443 | // +------------------------+ |
| 1444 | // | 6. padding | |
| 1445 | // +------------------------+ |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1446 | // | 7. allocas (variable) | |
| 1447 | // +------------------------+ |
| 1448 | // | 8. padding | |
| 1449 | // +------------------------+ |
| 1450 | // | 9. out args | |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1451 | // +------------------------+ <--- StackPointer |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1452 | // |
| 1453 | // The following variables record the size in bytes of the given areas: |
| 1454 | // * PreservedRegsSizeBytes: area 1 |
| 1455 | // * SpillAreaPaddingBytes: area 2 |
| 1456 | // * GlobalsSize: area 3 |
| 1457 | // * GlobalsAndSubsequentPaddingSize: areas 3 - 4 |
| 1458 | // * LocalsSpillAreaSize: area 5 |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1459 | // * SpillAreaSizeBytes: areas 2 - 6, and 9 |
| 1460 | // * MaxOutArgsSizeBytes: area 9 |
| 1461 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1462 | // Determine stack frame offsets for each Variable without a register |
| 1463 | // assignment. This can be done as one variable per stack slot. Or, do |
| 1464 | // coalescing by running the register allocator again with an infinite set of |
| 1465 | // registers (as a side effect, this gives variables a second chance at |
| 1466 | // physical register assignment). |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1467 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1468 | // A middle ground approach is to leverage sparsity and allocate one block of |
| 1469 | // space on the frame for globals (variables with multi-block lifetime), and |
| 1470 | // one block to share for locals (single-block lifetime). |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1471 | |
| 1472 | Context.init(Node); |
| 1473 | Context.setInsertPoint(Context.getCur()); |
| 1474 | |
John Porto | e82b560 | 2016-02-24 15:58:55 -0800 | [diff] [blame] | 1475 | SmallBitVector CalleeSaves = getRegisterSet(RegSet_CalleeSave, RegSet_None); |
| 1476 | RegsUsed = SmallBitVector(CalleeSaves.size()); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1477 | VarList SortedSpilledVariables; |
| 1478 | size_t GlobalsSize = 0; |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1479 | // If there is a separate locals area, this represents that area. Otherwise |
| 1480 | // it counts any variable not counted by GlobalsSize. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1481 | SpillAreaSizeBytes = 0; |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1482 | // If there is a separate locals area, this specifies the alignment for it. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1483 | uint32_t LocalsSlotsAlignmentBytes = 0; |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1484 | // The entire spill locations area gets aligned to largest natural alignment |
| 1485 | // of the variables that have a spill slot. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1486 | uint32_t SpillAreaAlignmentBytes = 0; |
| 1487 | // For now, we don't have target-specific variables that need special |
| 1488 | // treatment (no stack-slot-linked SpillVariable type). |
John Porto | e0b829f | 2015-09-28 09:50:48 -0700 | [diff] [blame] | 1489 | std::function<bool(Variable *)> TargetVarHook = [](Variable *Var) { |
| 1490 | static constexpr bool AssignStackSlot = false; |
| 1491 | static constexpr bool DontAssignStackSlot = !AssignStackSlot; |
| 1492 | if (llvm::isa<Variable64On32>(Var)) { |
| 1493 | return DontAssignStackSlot; |
| 1494 | } |
| 1495 | return AssignStackSlot; |
| 1496 | }; |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1497 | |
| 1498 | // Compute the list of spilled variables and bounds for GlobalsSize, etc. |
| 1499 | getVarStackSlotParams(SortedSpilledVariables, RegsUsed, &GlobalsSize, |
| 1500 | &SpillAreaSizeBytes, &SpillAreaAlignmentBytes, |
| 1501 | &LocalsSlotsAlignmentBytes, TargetVarHook); |
| 1502 | uint32_t LocalsSpillAreaSize = SpillAreaSizeBytes; |
| 1503 | SpillAreaSizeBytes += GlobalsSize; |
| 1504 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1505 | // Add push instructions for preserved registers. On ARM, "push" can push a |
| 1506 | // whole list of GPRs via a bitmask (0-15). Unlike x86, ARM also has |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1507 | // callee-saved float/vector registers. |
| 1508 | // |
| 1509 | // The "vpush" instruction can handle a whole list of float/vector registers, |
| 1510 | // but it only handles contiguous sequences of registers by specifying the |
| 1511 | // start and the length. |
| 1512 | PreservedGPRs.reserve(CalleeSaves.size()); |
| 1513 | PreservedSRegs.reserve(CalleeSaves.size()); |
| 1514 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1515 | // Consider FP and LR as callee-save / used as needed. |
| 1516 | if (UsesFramePointer) { |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1517 | if (RegsUsed[RegARM32::Reg_fp]) { |
| 1518 | llvm::report_fatal_error("Frame pointer has been used."); |
| 1519 | } |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1520 | CalleeSaves[RegARM32::Reg_fp] = true; |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1521 | RegsUsed[RegARM32::Reg_fp] = true; |
| 1522 | } |
| 1523 | if (!MaybeLeafFunc) { |
| 1524 | CalleeSaves[RegARM32::Reg_lr] = true; |
| 1525 | RegsUsed[RegARM32::Reg_lr] = true; |
| 1526 | } |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1527 | |
| 1528 | // Make two passes over the used registers. The first pass records all the |
| 1529 | // used registers -- and their aliases. Then, we figure out which GPRs and |
| 1530 | // VFP S registers should be saved. We don't bother saving D/Q registers |
| 1531 | // because their uses are recorded as S regs uses. |
John Porto | e82b560 | 2016-02-24 15:58:55 -0800 | [diff] [blame] | 1532 | SmallBitVector ToPreserve(RegARM32::Reg_NUM); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1533 | for (SizeT i = 0; i < CalleeSaves.size(); ++i) { |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1534 | if (NeedSandboxing && i == RegARM32::Reg_r9) { |
| 1535 | // r9 is never updated in sandboxed code. |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 1536 | continue; |
| 1537 | } |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1538 | if (CalleeSaves[i] && RegsUsed[i]) { |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1539 | ToPreserve |= RegisterAliases[i]; |
| 1540 | } |
| 1541 | } |
| 1542 | |
| 1543 | uint32_t NumCallee = 0; |
| 1544 | size_t PreservedRegsSizeBytes = 0; |
| 1545 | |
| 1546 | // RegClasses is a tuple of |
| 1547 | // |
| 1548 | // <First Register in Class, Last Register in Class, Vector of Save Registers> |
| 1549 | // |
| 1550 | // We use this tuple to figure out which register we should push/pop during |
| 1551 | // prolog/epilog. |
| 1552 | using RegClassType = std::tuple<uint32_t, uint32_t, VarList *>; |
| 1553 | const RegClassType RegClasses[] = { |
| 1554 | RegClassType(RegARM32::Reg_GPR_First, RegARM32::Reg_GPR_Last, |
| 1555 | &PreservedGPRs), |
| 1556 | RegClassType(RegARM32::Reg_SREG_First, RegARM32::Reg_SREG_Last, |
| 1557 | &PreservedSRegs)}; |
| 1558 | for (const auto &RegClass : RegClasses) { |
| 1559 | const uint32_t FirstRegInClass = std::get<0>(RegClass); |
| 1560 | const uint32_t LastRegInClass = std::get<1>(RegClass); |
| 1561 | VarList *const PreservedRegsInClass = std::get<2>(RegClass); |
| 1562 | for (uint32_t Reg = FirstRegInClass; Reg <= LastRegInClass; ++Reg) { |
| 1563 | if (!ToPreserve[Reg]) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1564 | continue; |
| 1565 | } |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1566 | ++NumCallee; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1567 | Variable *PhysicalRegister = getPhysicalRegister(RegNumT::fromInt(Reg)); |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 1568 | PreservedRegsSizeBytes += |
| 1569 | typeWidthInBytesOnStack(PhysicalRegister->getType()); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1570 | PreservedRegsInClass->push_back(PhysicalRegister); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1571 | } |
| 1572 | } |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1573 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1574 | Ctx->statsUpdateRegistersSaved(NumCallee); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1575 | if (!PreservedSRegs.empty()) |
| 1576 | _push(PreservedSRegs); |
| 1577 | if (!PreservedGPRs.empty()) |
| 1578 | _push(PreservedGPRs); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1579 | |
| 1580 | // Generate "mov FP, SP" if needed. |
| 1581 | if (UsesFramePointer) { |
| 1582 | Variable *FP = getPhysicalRegister(RegARM32::Reg_fp); |
| 1583 | Variable *SP = getPhysicalRegister(RegARM32::Reg_sp); |
| 1584 | _mov(FP, SP); |
| 1585 | // Keep FP live for late-stage liveness analysis (e.g. asm-verbose mode). |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 1586 | Context.insert<InstFakeUse>(FP); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1587 | } |
| 1588 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1589 | // Align the variables area. SpillAreaPaddingBytes is the size of the region |
| 1590 | // after the preserved registers and before the spill areas. |
| 1591 | // LocalsSlotsPaddingBytes is the amount of padding between the globals and |
| 1592 | // locals area if they are separate. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1593 | assert(SpillAreaAlignmentBytes <= ARM32_STACK_ALIGNMENT_BYTES); |
| 1594 | assert(LocalsSlotsAlignmentBytes <= SpillAreaAlignmentBytes); |
| 1595 | uint32_t SpillAreaPaddingBytes = 0; |
| 1596 | uint32_t LocalsSlotsPaddingBytes = 0; |
| 1597 | alignStackSpillAreas(PreservedRegsSizeBytes, SpillAreaAlignmentBytes, |
| 1598 | GlobalsSize, LocalsSlotsAlignmentBytes, |
| 1599 | &SpillAreaPaddingBytes, &LocalsSlotsPaddingBytes); |
| 1600 | SpillAreaSizeBytes += SpillAreaPaddingBytes + LocalsSlotsPaddingBytes; |
| 1601 | uint32_t GlobalsAndSubsequentPaddingSize = |
| 1602 | GlobalsSize + LocalsSlotsPaddingBytes; |
| 1603 | |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1604 | // Adds the out args space to the stack, and align SP if necessary. |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1605 | if (!NeedsStackAlignment) { |
| 1606 | SpillAreaSizeBytes += MaxOutArgsSizeBytes; |
| 1607 | } else { |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1608 | uint32_t StackOffset = PreservedRegsSizeBytes; |
| 1609 | uint32_t StackSize = applyStackAlignment(StackOffset + SpillAreaSizeBytes); |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1610 | StackSize = applyStackAlignment(StackSize + MaxOutArgsSizeBytes); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1611 | SpillAreaSizeBytes = StackSize - StackOffset; |
| 1612 | } |
| 1613 | |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1614 | // Combine fixed alloca with SpillAreaSize. |
| 1615 | SpillAreaSizeBytes += FixedAllocaSizeBytes; |
| 1616 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1617 | // Generate "sub sp, SpillAreaSizeBytes" |
| 1618 | if (SpillAreaSizeBytes) { |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1619 | // Use the scratch register if needed to legalize the immediate. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1620 | Operand *SubAmount = legalize(Ctx->getConstantInt32(SpillAreaSizeBytes), |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1621 | Legal_Reg | Legal_Flex, getReservedTmpReg()); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1622 | Sandboxer(this).sub_sp(SubAmount); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1623 | if (FixedAllocaAlignBytes > ARM32_STACK_ALIGNMENT_BYTES) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1624 | Sandboxer(this).align_sp(FixedAllocaAlignBytes); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1625 | } |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1626 | } |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1627 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1628 | Ctx->statsUpdateFrameBytes(SpillAreaSizeBytes); |
| 1629 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1630 | // Fill in stack offsets for stack args, and copy args into registers for |
| 1631 | // those that were register-allocated. Args are pushed right to left, so |
| 1632 | // Arg[0] is closest to the stack/frame pointer. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1633 | Variable *FramePtr = getPhysicalRegister(getFrameOrStackReg()); |
| 1634 | size_t BasicFrameOffset = PreservedRegsSizeBytes; |
| 1635 | if (!UsesFramePointer) |
| 1636 | BasicFrameOffset += SpillAreaSizeBytes; |
| 1637 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 1638 | materializeGotAddr(Node); |
| 1639 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1640 | const VarList &Args = Func->getArgs(); |
| 1641 | size_t InArgsSizeBytes = 0; |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 1642 | TargetARM32::CallingConv CC; |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1643 | for (Variable *Arg : Args) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1644 | RegNumT DummyReg; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1645 | const Type Ty = Arg->getType(); |
| 1646 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1647 | // Skip arguments passed in registers. |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1648 | if (isScalarIntegerType(Ty)) { |
| 1649 | if (CC.argInGPR(Ty, &DummyReg)) { |
| 1650 | continue; |
| 1651 | } |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 1652 | } else { |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1653 | if (CC.argInVFP(Ty, &DummyReg)) { |
| 1654 | continue; |
| 1655 | } |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1656 | } |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 1657 | finishArgumentLowering(Arg, FramePtr, BasicFrameOffset, &InArgsSizeBytes); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1658 | } |
| 1659 | |
| 1660 | // Fill in stack offsets for locals. |
| 1661 | assignVarStackSlots(SortedSpilledVariables, SpillAreaPaddingBytes, |
| 1662 | SpillAreaSizeBytes, GlobalsAndSubsequentPaddingSize, |
| 1663 | UsesFramePointer); |
| 1664 | this->HasComputedFrame = true; |
| 1665 | |
Jim Stichnoth | 20b71f5 | 2015-06-24 15:52:24 -0700 | [diff] [blame] | 1666 | if (BuildDefs::dump() && Func->isVerbose(IceV_Frame)) { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1667 | OstreamLocker _(Func->getContext()); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1668 | Ostream &Str = Func->getContext()->getStrDump(); |
| 1669 | |
| 1670 | Str << "Stack layout:\n"; |
| 1671 | uint32_t SPAdjustmentPaddingSize = |
| 1672 | SpillAreaSizeBytes - LocalsSpillAreaSize - |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1673 | GlobalsAndSubsequentPaddingSize - SpillAreaPaddingBytes - |
| 1674 | MaxOutArgsSizeBytes; |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1675 | Str << " in-args = " << InArgsSizeBytes << " bytes\n" |
| 1676 | << " preserved registers = " << PreservedRegsSizeBytes << " bytes\n" |
| 1677 | << " spill area padding = " << SpillAreaPaddingBytes << " bytes\n" |
| 1678 | << " globals spill area = " << GlobalsSize << " bytes\n" |
| 1679 | << " globals-locals spill areas intermediate padding = " |
| 1680 | << GlobalsAndSubsequentPaddingSize - GlobalsSize << " bytes\n" |
| 1681 | << " locals spill area = " << LocalsSpillAreaSize << " bytes\n" |
| 1682 | << " SP alignment padding = " << SPAdjustmentPaddingSize << " bytes\n"; |
| 1683 | |
| 1684 | Str << "Stack details:\n" |
| 1685 | << " SP adjustment = " << SpillAreaSizeBytes << " bytes\n" |
| 1686 | << " spill area alignment = " << SpillAreaAlignmentBytes << " bytes\n" |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 1687 | << " outgoing args size = " << MaxOutArgsSizeBytes << " bytes\n" |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1688 | << " locals spill area alignment = " << LocalsSlotsAlignmentBytes |
| 1689 | << " bytes\n" |
| 1690 | << " is FP based = " << UsesFramePointer << "\n"; |
| 1691 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1692 | } |
| 1693 | |
| 1694 | void TargetARM32::addEpilog(CfgNode *Node) { |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1695 | InstList &Insts = Node->getInsts(); |
| 1696 | InstList::reverse_iterator RI, E; |
| 1697 | for (RI = Insts.rbegin(), E = Insts.rend(); RI != E; ++RI) { |
| 1698 | if (llvm::isa<InstARM32Ret>(*RI)) |
| 1699 | break; |
| 1700 | } |
| 1701 | if (RI == E) |
| 1702 | return; |
| 1703 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1704 | // Convert the reverse_iterator position into its corresponding (forward) |
| 1705 | // iterator position. |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1706 | InstList::iterator InsertPoint = RI.base(); |
| 1707 | --InsertPoint; |
| 1708 | Context.init(Node); |
| 1709 | Context.setInsertPoint(InsertPoint); |
| 1710 | |
| 1711 | Variable *SP = getPhysicalRegister(RegARM32::Reg_sp); |
| 1712 | if (UsesFramePointer) { |
| 1713 | Variable *FP = getPhysicalRegister(RegARM32::Reg_fp); |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1714 | // For late-stage liveness analysis (e.g. asm-verbose mode), adding a fake |
| 1715 | // use of SP before the assignment of SP=FP keeps previous SP adjustments |
| 1716 | // from being dead-code eliminated. |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 1717 | Context.insert<InstFakeUse>(SP); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1718 | Sandboxer(this).reset_sp(FP); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1719 | } else { |
| 1720 | // add SP, SpillAreaSizeBytes |
| 1721 | if (SpillAreaSizeBytes) { |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1722 | // Use the scratch register if needed to legalize the immediate. |
| 1723 | Operand *AddAmount = |
| 1724 | legalize(Ctx->getConstantInt32(SpillAreaSizeBytes), |
| 1725 | Legal_Reg | Legal_Flex, getReservedTmpReg()); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1726 | Sandboxer(this).add_sp(AddAmount); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1727 | } |
| 1728 | } |
| 1729 | |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1730 | if (!PreservedGPRs.empty()) |
| 1731 | _pop(PreservedGPRs); |
| 1732 | if (!PreservedSRegs.empty()) |
| 1733 | _pop(PreservedSRegs); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1734 | |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 1735 | if (!getFlags().getUseSandboxing()) |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1736 | return; |
| 1737 | |
| 1738 | // Change the original ret instruction into a sandboxed return sequence. |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1739 | // |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1740 | // bundle_lock |
| 1741 | // bic lr, #0xc000000f |
| 1742 | // bx lr |
| 1743 | // bundle_unlock |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 1744 | // |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1745 | // This isn't just aligning to the getBundleAlignLog2Bytes(). It needs to |
| 1746 | // restrict to the lower 1GB as well. |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1747 | Variable *LR = getPhysicalRegister(RegARM32::Reg_lr); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1748 | Variable *RetValue = nullptr; |
| 1749 | if (RI->getSrcSize()) |
| 1750 | RetValue = llvm::cast<Variable>(RI->getSrc(0)); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1751 | |
| 1752 | Sandboxer(this).ret(LR, RetValue); |
| 1753 | |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 1754 | RI->setDeleted(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 1755 | } |
| 1756 | |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1757 | bool TargetARM32::isLegalMemOffset(Type Ty, int32_t Offset) const { |
| 1758 | constexpr bool ZeroExt = false; |
| 1759 | return OperandARM32Mem::canHoldOffset(Ty, ZeroExt, Offset); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1760 | } |
| 1761 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1762 | Variable *TargetARM32::PostLoweringLegalizer::newBaseRegister( |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1763 | Variable *Base, int32_t Offset, RegNumT ScratchRegNum) { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 1764 | // Legalize will likely need a movw/movt combination, but if the top bits are |
| 1765 | // all 0 from negating the offset and subtracting, we could use that instead. |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1766 | const bool ShouldSub = Offset != 0 && (-Offset & 0xFFFF0000) == 0; |
| 1767 | Variable *ScratchReg = Target->makeReg(IceType_i32, ScratchRegNum); |
| 1768 | if (ShouldSub) { |
| 1769 | Operand *OffsetVal = |
| 1770 | Target->legalize(Target->Ctx->getConstantInt32(-Offset), |
| 1771 | Legal_Reg | Legal_Flex, ScratchRegNum); |
| 1772 | Target->_sub(ScratchReg, Base, OffsetVal); |
| 1773 | } else { |
| 1774 | Operand *OffsetVal = |
| 1775 | Target->legalize(Target->Ctx->getConstantInt32(Offset), |
| 1776 | Legal_Reg | Legal_Flex, ScratchRegNum); |
| 1777 | Target->_add(ScratchReg, Base, OffsetVal); |
| 1778 | } |
| 1779 | |
| 1780 | if (ScratchRegNum == Target->getReservedTmpReg()) { |
| 1781 | const bool BaseIsStackOrFramePtr = |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1782 | Base->getRegNum() == Target->getFrameOrStackReg(); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1783 | // There is currently no code path that would trigger this assertion, so we |
| 1784 | // leave this assertion here in case it is ever violated. This is not a |
| 1785 | // fatal error (thus the use of assert() and not llvm::report_fatal_error) |
| 1786 | // as the program compiled by subzero will still work correctly. |
| 1787 | assert(BaseIsStackOrFramePtr); |
| 1788 | // Side-effect: updates TempBase to reflect the new Temporary. |
| 1789 | if (BaseIsStackOrFramePtr) { |
| 1790 | TempBaseReg = ScratchReg; |
| 1791 | TempBaseOffset = Offset; |
| 1792 | } else { |
| 1793 | TempBaseReg = nullptr; |
| 1794 | TempBaseOffset = 0; |
| 1795 | } |
| 1796 | } |
| 1797 | |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1798 | return ScratchReg; |
| 1799 | } |
| 1800 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1801 | OperandARM32Mem *TargetARM32::PostLoweringLegalizer::createMemOperand( |
| 1802 | Type Ty, Variable *Base, int32_t Offset, bool AllowOffsets) { |
| 1803 | assert(!Base->isRematerializable()); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1804 | if (Offset == 0 || (AllowOffsets && Target->isLegalMemOffset(Ty, Offset))) { |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1805 | return OperandARM32Mem::create( |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1806 | Target->Func, Ty, Base, |
| 1807 | llvm::cast<ConstantInteger32>(Target->Ctx->getConstantInt32(Offset)), |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1808 | OperandARM32Mem::Offset); |
| 1809 | } |
| 1810 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1811 | if (!AllowOffsets || TempBaseReg == nullptr) { |
| 1812 | newBaseRegister(Base, Offset, Target->getReservedTmpReg()); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1813 | } |
| 1814 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1815 | int32_t OffsetDiff = Offset - TempBaseOffset; |
| 1816 | assert(AllowOffsets || OffsetDiff == 0); |
| 1817 | |
| 1818 | if (!Target->isLegalMemOffset(Ty, OffsetDiff)) { |
| 1819 | newBaseRegister(Base, Offset, Target->getReservedTmpReg()); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1820 | OffsetDiff = 0; |
| 1821 | } |
| 1822 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1823 | assert(!TempBaseReg->isRematerializable()); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1824 | return OperandARM32Mem::create( |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1825 | Target->Func, Ty, TempBaseReg, |
| 1826 | llvm::cast<ConstantInteger32>(Target->Ctx->getConstantInt32(OffsetDiff)), |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1827 | OperandARM32Mem::Offset); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1828 | } |
| 1829 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1830 | void TargetARM32::PostLoweringLegalizer::resetTempBaseIfClobberedBy( |
| 1831 | const Inst *Instr) { |
| 1832 | bool ClobbersTempBase = false; |
| 1833 | if (TempBaseReg != nullptr) { |
| 1834 | Variable *Dest = Instr->getDest(); |
| 1835 | if (llvm::isa<InstARM32Call>(Instr)) { |
| 1836 | // The following assertion is an invariant, so we remove it from the if |
| 1837 | // test. If the invariant is ever broken/invalidated/changed, remember |
| 1838 | // to add it back to the if condition. |
| 1839 | assert(TempBaseReg->getRegNum() == Target->getReservedTmpReg()); |
| 1840 | // The linker may need to clobber IP if the call is too far from PC. Thus, |
| 1841 | // we assume IP will be overwritten. |
| 1842 | ClobbersTempBase = true; |
| 1843 | } else if (Dest != nullptr && |
| 1844 | Dest->getRegNum() == TempBaseReg->getRegNum()) { |
| 1845 | // Register redefinition. |
| 1846 | ClobbersTempBase = true; |
| 1847 | } |
| 1848 | } |
| 1849 | |
| 1850 | if (ClobbersTempBase) { |
| 1851 | TempBaseReg = nullptr; |
| 1852 | TempBaseOffset = 0; |
| 1853 | } |
| 1854 | } |
| 1855 | |
| 1856 | void TargetARM32::PostLoweringLegalizer::legalizeMov(InstARM32Mov *MovInstr) { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1857 | Variable *Dest = MovInstr->getDest(); |
| 1858 | assert(Dest != nullptr); |
| 1859 | Type DestTy = Dest->getType(); |
| 1860 | assert(DestTy != IceType_i64); |
| 1861 | |
| 1862 | Operand *Src = MovInstr->getSrc(0); |
| 1863 | Type SrcTy = Src->getType(); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1864 | (void)SrcTy; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1865 | assert(SrcTy != IceType_i64); |
| 1866 | |
| 1867 | if (MovInstr->isMultiDest() || MovInstr->isMultiSource()) |
| 1868 | return; |
| 1869 | |
| 1870 | bool Legalized = false; |
| 1871 | if (!Dest->hasReg()) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1872 | auto *SrcR = llvm::cast<Variable>(Src); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1873 | assert(SrcR->hasReg()); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1874 | assert(!SrcR->isRematerializable()); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1875 | const int32_t Offset = Dest->getStackOffset(); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1876 | // This is a _mov(Mem(), Variable), i.e., a store. |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1877 | TargetARM32::Sandboxer(Target) |
| 1878 | .str(SrcR, createMemOperand(DestTy, StackOrFrameReg, Offset), |
| 1879 | MovInstr->getPredicate()); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1880 | // _str() does not have a Dest, so we add a fake-def(Dest). |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 1881 | Target->Context.insert<InstFakeDef>(Dest); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1882 | Legalized = true; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1883 | } else if (auto *Var = llvm::dyn_cast<Variable>(Src)) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1884 | if (Var->isRematerializable()) { |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1885 | // This is equivalent to an x86 _lea(RematOffset(%esp/%ebp), Variable). |
| 1886 | |
| 1887 | // ExtraOffset is only needed for frame-pointer based frames as we have |
| 1888 | // to account for spill storage. |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1889 | const int32_t ExtraOffset = (Var->getRegNum() == Target->getFrameReg()) |
| 1890 | ? Target->getFrameFixedAllocaOffset() |
| 1891 | : 0; |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1892 | |
| 1893 | const int32_t Offset = Var->getStackOffset() + ExtraOffset; |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1894 | Variable *Base = Target->getPhysicalRegister(Var->getRegNum()); |
| 1895 | Variable *T = newBaseRegister(Base, Offset, Dest->getRegNum()); |
| 1896 | Target->_mov(Dest, T); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 1897 | Legalized = true; |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1898 | } else { |
| 1899 | if (!Var->hasReg()) { |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1900 | // This is a _mov(Variable, Mem()), i.e., a load. |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1901 | const int32_t Offset = Var->getStackOffset(); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1902 | TargetARM32::Sandboxer(Target) |
| 1903 | .ldr(Dest, createMemOperand(DestTy, StackOrFrameReg, Offset), |
| 1904 | MovInstr->getPredicate()); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 1905 | Legalized = true; |
| 1906 | } |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1907 | } |
| 1908 | } |
| 1909 | |
| 1910 | if (Legalized) { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 1911 | if (MovInstr->isDestRedefined()) { |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1912 | Target->_set_dest_redefined(); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 1913 | } |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 1914 | MovInstr->setDeleted(); |
| 1915 | } |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 1916 | } |
| 1917 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1918 | // ARM32 address modes: |
| 1919 | // ld/st i[8|16|32]: [reg], [reg +/- imm12], [pc +/- imm12], |
| 1920 | // [reg +/- reg << shamt5] |
| 1921 | // ld/st f[32|64] : [reg], [reg +/- imm8] , [pc +/- imm8] |
| 1922 | // ld/st vectors : [reg] |
| 1923 | // |
| 1924 | // For now, we don't handle address modes with Relocatables. |
| 1925 | namespace { |
| 1926 | // MemTraits contains per-type valid address mode information. |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 1927 | #define X(tag, elementty, int_width, fp_width, uvec_width, svec_width, sbits, \ |
| 1928 | ubits, rraddr, shaddr) \ |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1929 | static_assert(!(shaddr) || rraddr, "Check ICETYPEARM32_TABLE::" #tag); |
| 1930 | ICETYPEARM32_TABLE |
| 1931 | #undef X |
| 1932 | |
| 1933 | static const struct { |
| 1934 | int32_t ValidImmMask; |
| 1935 | bool CanHaveImm; |
| 1936 | bool CanHaveIndex; |
| 1937 | bool CanHaveShiftedIndex; |
| 1938 | } MemTraits[] = { |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 1939 | #define X(tag, elementty, int_width, fp_width, uvec_width, svec_width, sbits, \ |
| 1940 | ubits, rraddr, shaddr) \ |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1941 | { (1 << ubits) - 1, (ubits) > 0, rraddr, shaddr, } \ |
| 1942 | , |
| 1943 | ICETYPEARM32_TABLE |
| 1944 | #undef X |
| 1945 | }; |
| 1946 | static constexpr SizeT MemTraitsSize = llvm::array_lengthof(MemTraits); |
| 1947 | } // end of anonymous namespace |
| 1948 | |
| 1949 | OperandARM32Mem * |
| 1950 | TargetARM32::PostLoweringLegalizer::legalizeMemOperand(OperandARM32Mem *Mem, |
| 1951 | bool AllowOffsets) { |
| 1952 | assert(!Mem->isRegReg() || !Mem->getIndex()->isRematerializable()); |
| 1953 | assert( |
| 1954 | Mem->isRegReg() || |
| 1955 | Target->isLegalMemOffset(Mem->getType(), Mem->getOffset()->getValue())); |
| 1956 | |
| 1957 | bool Legalized = false; |
| 1958 | Variable *Base = Mem->getBase(); |
| 1959 | int32_t Offset = Mem->isRegReg() ? 0 : Mem->getOffset()->getValue(); |
| 1960 | if (Base->isRematerializable()) { |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 1961 | const int32_t ExtraOffset = (Base->getRegNum() == Target->getFrameReg()) |
| 1962 | ? Target->getFrameFixedAllocaOffset() |
| 1963 | : 0; |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1964 | Offset += Base->getStackOffset() + ExtraOffset; |
| 1965 | Base = Target->getPhysicalRegister(Base->getRegNum()); |
| 1966 | assert(!Base->isRematerializable()); |
| 1967 | Legalized = true; |
| 1968 | } |
| 1969 | |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1970 | if (!Legalized && !Target->NeedSandboxing) { |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1971 | return nullptr; |
| 1972 | } |
| 1973 | |
| 1974 | if (!Mem->isRegReg()) { |
| 1975 | return createMemOperand(Mem->getType(), Base, Offset, AllowOffsets); |
| 1976 | } |
| 1977 | |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 1978 | if (Target->NeedSandboxing) { |
| 1979 | llvm::report_fatal_error("Reg-Reg address mode is not allowed."); |
| 1980 | } |
| 1981 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 1982 | assert(MemTraits[Mem->getType()].CanHaveIndex); |
| 1983 | |
| 1984 | if (Offset != 0) { |
| 1985 | if (TempBaseReg == nullptr) { |
| 1986 | Base = newBaseRegister(Base, Offset, Target->getReservedTmpReg()); |
| 1987 | } else { |
| 1988 | uint32_t Imm8, Rotate; |
| 1989 | const int32_t OffsetDiff = Offset - TempBaseOffset; |
| 1990 | if (OffsetDiff == 0) { |
| 1991 | Base = TempBaseReg; |
| 1992 | } else if (OperandARM32FlexImm::canHoldImm(OffsetDiff, &Rotate, &Imm8)) { |
| 1993 | auto *OffsetDiffF = OperandARM32FlexImm::create( |
| 1994 | Target->Func, IceType_i32, Imm8, Rotate); |
| 1995 | Target->_add(TempBaseReg, TempBaseReg, OffsetDiffF); |
| 1996 | TempBaseOffset += OffsetDiff; |
| 1997 | Base = TempBaseReg; |
| 1998 | } else if (OperandARM32FlexImm::canHoldImm(-OffsetDiff, &Rotate, &Imm8)) { |
| 1999 | auto *OffsetDiffF = OperandARM32FlexImm::create( |
| 2000 | Target->Func, IceType_i32, Imm8, Rotate); |
| 2001 | Target->_sub(TempBaseReg, TempBaseReg, OffsetDiffF); |
| 2002 | TempBaseOffset += OffsetDiff; |
| 2003 | Base = TempBaseReg; |
| 2004 | } else { |
| 2005 | Base = newBaseRegister(Base, Offset, Target->getReservedTmpReg()); |
| 2006 | } |
| 2007 | } |
| 2008 | } |
| 2009 | |
| 2010 | return OperandARM32Mem::create(Target->Func, Mem->getType(), Base, |
| 2011 | Mem->getIndex(), Mem->getShiftOp(), |
| 2012 | Mem->getShiftAmt(), Mem->getAddrMode()); |
| 2013 | } |
| 2014 | |
| 2015 | void TargetARM32::postLowerLegalization() { |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2016 | // If a stack variable's frame offset doesn't fit, convert from: |
| 2017 | // ldr X, OFF[SP] |
| 2018 | // to: |
| 2019 | // movw/movt TMP, OFF_PART |
| 2020 | // add TMP, TMP, SP |
| 2021 | // ldr X, OFF_MORE[TMP] |
| 2022 | // |
| 2023 | // This is safe because we have reserved TMP, and add for ARM does not |
| 2024 | // clobber the flags register. |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2025 | Func->dump("Before postLowerLegalization"); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2026 | assert(hasComputedFrame()); |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2027 | // Do a fairly naive greedy clustering for now. Pick the first stack slot |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2028 | // that's out of bounds and make a new base reg using the architecture's temp |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2029 | // register. If that works for the next slot, then great. Otherwise, create a |
| 2030 | // new base register, clobbering the previous base register. Never share a |
| 2031 | // base reg across different basic blocks. This isn't ideal if local and |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2032 | // multi-block variables are far apart and their references are interspersed. |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2033 | // It may help to be more coordinated about assign stack slot numbers and may |
| 2034 | // help to assign smaller offsets to higher-weight variables so that they |
| 2035 | // don't depend on this legalization. |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2036 | for (CfgNode *Node : Func->getNodes()) { |
| 2037 | Context.init(Node); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2038 | // One legalizer per basic block, otherwise we would share the Temporary |
| 2039 | // Base Register between basic blocks. |
| 2040 | PostLoweringLegalizer Legalizer(this); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2041 | while (!Context.atEnd()) { |
| 2042 | PostIncrLoweringContext PostIncrement(Context); |
Jim Stichnoth | f5fdd23 | 2016-05-09 12:24:36 -0700 | [diff] [blame] | 2043 | Inst *CurInstr = iteratorToInst(Context.getCur()); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 2044 | |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2045 | // Check if the previous TempBaseReg is clobbered, and reset if needed. |
| 2046 | Legalizer.resetTempBaseIfClobberedBy(CurInstr); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 2047 | |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 2048 | if (auto *MovInstr = llvm::dyn_cast<InstARM32Mov>(CurInstr)) { |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2049 | Legalizer.legalizeMov(MovInstr); |
| 2050 | } else if (auto *LdrInstr = llvm::dyn_cast<InstARM32Ldr>(CurInstr)) { |
| 2051 | if (OperandARM32Mem *LegalMem = Legalizer.legalizeMemOperand( |
| 2052 | llvm::cast<OperandARM32Mem>(LdrInstr->getSrc(0)))) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 2053 | Sandboxer(this) |
| 2054 | .ldr(CurInstr->getDest(), LegalMem, LdrInstr->getPredicate()); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2055 | CurInstr->setDeleted(); |
| 2056 | } |
| 2057 | } else if (auto *LdrexInstr = llvm::dyn_cast<InstARM32Ldrex>(CurInstr)) { |
| 2058 | constexpr bool DisallowOffsetsBecauseLdrex = false; |
| 2059 | if (OperandARM32Mem *LegalMem = Legalizer.legalizeMemOperand( |
| 2060 | llvm::cast<OperandARM32Mem>(LdrexInstr->getSrc(0)), |
| 2061 | DisallowOffsetsBecauseLdrex)) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 2062 | Sandboxer(this) |
| 2063 | .ldrex(CurInstr->getDest(), LegalMem, LdrexInstr->getPredicate()); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2064 | CurInstr->setDeleted(); |
| 2065 | } |
| 2066 | } else if (auto *StrInstr = llvm::dyn_cast<InstARM32Str>(CurInstr)) { |
| 2067 | if (OperandARM32Mem *LegalMem = Legalizer.legalizeMemOperand( |
| 2068 | llvm::cast<OperandARM32Mem>(StrInstr->getSrc(1)))) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 2069 | Sandboxer(this).str(llvm::cast<Variable>(CurInstr->getSrc(0)), |
| 2070 | LegalMem, StrInstr->getPredicate()); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2071 | CurInstr->setDeleted(); |
| 2072 | } |
| 2073 | } else if (auto *StrexInstr = llvm::dyn_cast<InstARM32Strex>(CurInstr)) { |
| 2074 | constexpr bool DisallowOffsetsBecauseStrex = false; |
| 2075 | if (OperandARM32Mem *LegalMem = Legalizer.legalizeMemOperand( |
| 2076 | llvm::cast<OperandARM32Mem>(StrexInstr->getSrc(1)), |
| 2077 | DisallowOffsetsBecauseStrex)) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 2078 | Sandboxer(this).strex(CurInstr->getDest(), |
| 2079 | llvm::cast<Variable>(CurInstr->getSrc(0)), |
| 2080 | LegalMem, StrexInstr->getPredicate()); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2081 | CurInstr->setDeleted(); |
| 2082 | } |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2083 | } |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2084 | |
| 2085 | // Sanity-check: the Legalizer will either have no Temp, or it will be |
| 2086 | // bound to IP. |
| 2087 | Legalizer.assertNoTempOrAssignedToIP(); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 2088 | } |
| 2089 | } |
| 2090 | } |
| 2091 | |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2092 | Operand *TargetARM32::loOperand(Operand *Operand) { |
| 2093 | assert(Operand->getType() == IceType_i64); |
| 2094 | if (Operand->getType() != IceType_i64) |
| 2095 | return Operand; |
Andrew Scull | 6d47bcd | 2015-09-17 17:10:05 -0700 | [diff] [blame] | 2096 | if (auto *Var64On32 = llvm::dyn_cast<Variable64On32>(Operand)) |
| 2097 | return Var64On32->getLo(); |
| 2098 | if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2099 | return Ctx->getConstantInt32(static_cast<uint32_t>(Const->getValue())); |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 2100 | if (auto *Mem = llvm::dyn_cast<OperandARM32Mem>(Operand)) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2101 | // Conservatively disallow memory operands with side-effects (pre/post |
| 2102 | // increment) in case of duplication. |
| 2103 | assert(Mem->getAddrMode() == OperandARM32Mem::Offset || |
| 2104 | Mem->getAddrMode() == OperandARM32Mem::NegOffset); |
| 2105 | if (Mem->isRegReg()) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2106 | Variable *IndexR = legalizeToReg(Mem->getIndex()); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2107 | return OperandARM32Mem::create(Func, IceType_i32, Mem->getBase(), IndexR, |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2108 | Mem->getShiftOp(), Mem->getShiftAmt(), |
| 2109 | Mem->getAddrMode()); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2110 | } else { |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 2111 | return OperandARM32Mem::create(Func, IceType_i32, Mem->getBase(), |
| 2112 | Mem->getOffset(), Mem->getAddrMode()); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2113 | } |
| 2114 | } |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 2115 | llvm::report_fatal_error("Unsupported operand type"); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2116 | return nullptr; |
| 2117 | } |
| 2118 | |
| 2119 | Operand *TargetARM32::hiOperand(Operand *Operand) { |
| 2120 | assert(Operand->getType() == IceType_i64); |
| 2121 | if (Operand->getType() != IceType_i64) |
| 2122 | return Operand; |
Andrew Scull | 6d47bcd | 2015-09-17 17:10:05 -0700 | [diff] [blame] | 2123 | if (auto *Var64On32 = llvm::dyn_cast<Variable64On32>(Operand)) |
| 2124 | return Var64On32->getHi(); |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 2125 | if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2126 | return Ctx->getConstantInt32( |
| 2127 | static_cast<uint32_t>(Const->getValue() >> 32)); |
| 2128 | } |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 2129 | if (auto *Mem = llvm::dyn_cast<OperandARM32Mem>(Operand)) { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2130 | // Conservatively disallow memory operands with side-effects in case of |
| 2131 | // duplication. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2132 | assert(Mem->getAddrMode() == OperandARM32Mem::Offset || |
| 2133 | Mem->getAddrMode() == OperandARM32Mem::NegOffset); |
| 2134 | const Type SplitType = IceType_i32; |
| 2135 | if (Mem->isRegReg()) { |
| 2136 | // We have to make a temp variable T, and add 4 to either Base or Index. |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2137 | // The Index may be shifted, so adding 4 can mean something else. Thus, |
| 2138 | // prefer T := Base + 4, and use T as the new Base. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2139 | Variable *Base = Mem->getBase(); |
| 2140 | Constant *Four = Ctx->getConstantInt32(4); |
| 2141 | Variable *NewBase = Func->makeVariable(Base->getType()); |
| 2142 | lowerArithmetic(InstArithmetic::create(Func, InstArithmetic::Add, NewBase, |
| 2143 | Base, Four)); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2144 | Variable *BaseR = legalizeToReg(NewBase); |
| 2145 | Variable *IndexR = legalizeToReg(Mem->getIndex()); |
| 2146 | return OperandARM32Mem::create(Func, SplitType, BaseR, IndexR, |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2147 | Mem->getShiftOp(), Mem->getShiftAmt(), |
| 2148 | Mem->getAddrMode()); |
| 2149 | } else { |
| 2150 | Variable *Base = Mem->getBase(); |
| 2151 | ConstantInteger32 *Offset = Mem->getOffset(); |
| 2152 | assert(!Utils::WouldOverflowAdd(Offset->getValue(), 4)); |
| 2153 | int32_t NextOffsetVal = Offset->getValue() + 4; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 2154 | constexpr bool ZeroExt = false; |
| 2155 | if (!OperandARM32Mem::canHoldOffset(SplitType, ZeroExt, NextOffsetVal)) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2156 | // We have to make a temp variable and add 4 to either Base or Offset. |
| 2157 | // If we add 4 to Offset, this will convert a non-RegReg addressing |
| 2158 | // mode into a RegReg addressing mode. Since NaCl sandboxing disallows |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2159 | // RegReg addressing modes, prefer adding to base and replacing |
| 2160 | // instead. Thus we leave the old offset alone. |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2161 | Constant *_4 = Ctx->getConstantInt32(4); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2162 | Variable *NewBase = Func->makeVariable(Base->getType()); |
| 2163 | lowerArithmetic(InstArithmetic::create(Func, InstArithmetic::Add, |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2164 | NewBase, Base, _4)); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2165 | Base = NewBase; |
| 2166 | } else { |
| 2167 | Offset = |
| 2168 | llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(NextOffsetVal)); |
| 2169 | } |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2170 | Variable *BaseR = legalizeToReg(Base); |
| 2171 | return OperandARM32Mem::create(Func, SplitType, BaseR, Offset, |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2172 | Mem->getAddrMode()); |
| 2173 | } |
| 2174 | } |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 2175 | llvm::report_fatal_error("Unsupported operand type"); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 2176 | return nullptr; |
| 2177 | } |
| 2178 | |
John Porto | e82b560 | 2016-02-24 15:58:55 -0800 | [diff] [blame] | 2179 | SmallBitVector TargetARM32::getRegisterSet(RegSetMask Include, |
| 2180 | RegSetMask Exclude) const { |
| 2181 | SmallBitVector Registers(RegARM32::Reg_NUM); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 2182 | |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 2183 | for (uint32_t i = 0; i < RegARM32::Reg_NUM; ++i) { |
Karl Schimpf | 57ec7df | 2016-01-15 08:11:00 -0800 | [diff] [blame] | 2184 | const auto &Entry = RegARM32::RegTable[i]; |
John Porto | 149999e | 2016-01-04 13:45:26 -0800 | [diff] [blame] | 2185 | if (Entry.Scratch && (Include & RegSet_CallerSave)) |
| 2186 | Registers[i] = true; |
| 2187 | if (Entry.Preserved && (Include & RegSet_CalleeSave)) |
| 2188 | Registers[i] = true; |
| 2189 | if (Entry.StackPtr && (Include & RegSet_StackPointer)) |
| 2190 | Registers[i] = true; |
| 2191 | if (Entry.FramePtr && (Include & RegSet_FramePointer)) |
| 2192 | Registers[i] = true; |
| 2193 | if (Entry.Scratch && (Exclude & RegSet_CallerSave)) |
| 2194 | Registers[i] = false; |
| 2195 | if (Entry.Preserved && (Exclude & RegSet_CalleeSave)) |
| 2196 | Registers[i] = false; |
| 2197 | if (Entry.StackPtr && (Exclude & RegSet_StackPointer)) |
| 2198 | Registers[i] = false; |
| 2199 | if (Entry.FramePtr && (Exclude & RegSet_FramePointer)) |
| 2200 | Registers[i] = false; |
| 2201 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 2202 | |
| 2203 | return Registers; |
| 2204 | } |
| 2205 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2206 | void TargetARM32::lowerAlloca(const InstAlloca *Instr) { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2207 | // Conservatively require the stack to be aligned. Some stack adjustment |
| 2208 | // operations implemented below assume that the stack is aligned before the |
| 2209 | // alloca. All the alloca code ensures that the stack alignment is preserved |
| 2210 | // after the alloca. The stack alignment restriction can be relaxed in some |
| 2211 | // cases. |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 2212 | NeedsStackAlignment = true; |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2213 | |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2214 | // For default align=0, set it to the real value 1, to avoid any |
| 2215 | // bit-manipulation problems below. |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2216 | const uint32_t AlignmentParam = std::max(1u, Instr->getAlignInBytes()); |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2217 | |
| 2218 | // LLVM enforces power of 2 alignment. |
| 2219 | assert(llvm::isPowerOf2_32(AlignmentParam)); |
| 2220 | assert(llvm::isPowerOf2_32(ARM32_STACK_ALIGNMENT_BYTES)); |
| 2221 | |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2222 | const uint32_t Alignment = |
| 2223 | std::max(AlignmentParam, ARM32_STACK_ALIGNMENT_BYTES); |
| 2224 | const bool OverAligned = Alignment > ARM32_STACK_ALIGNMENT_BYTES; |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 2225 | const bool OptM1 = getFlags().getOptLevel() == Opt_m1; |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2226 | const bool AllocaWithKnownOffset = Instr->getKnownFrameOffset(); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2227 | const bool UseFramePointer = |
| 2228 | hasFramePointer() || OverAligned || !AllocaWithKnownOffset || OptM1; |
| 2229 | |
| 2230 | if (UseFramePointer) |
| 2231 | setHasFramePointer(); |
| 2232 | |
| 2233 | Variable *SP = getPhysicalRegister(RegARM32::Reg_sp); |
| 2234 | if (OverAligned) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 2235 | Sandboxer(this).align_sp(Alignment); |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2236 | } |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2237 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2238 | Variable *Dest = Instr->getDest(); |
| 2239 | Operand *TotalSize = Instr->getSizeInBytes(); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2240 | |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2241 | if (const auto *ConstantTotalSize = |
| 2242 | llvm::dyn_cast<ConstantInteger32>(TotalSize)) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2243 | const uint32_t Value = |
| 2244 | Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); |
| 2245 | // Constant size alloca. |
| 2246 | if (!UseFramePointer) { |
| 2247 | // If we don't need a Frame Pointer, this alloca has a known offset to the |
| 2248 | // stack pointer. We don't need adjust the stack pointer, nor assign any |
| 2249 | // value to Dest, as Dest is rematerializable. |
| 2250 | assert(Dest->isRematerializable()); |
| 2251 | FixedAllocaSizeBytes += Value; |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 2252 | Context.insert<InstFakeDef>(Dest); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2253 | return; |
| 2254 | } |
| 2255 | |
| 2256 | // If a frame pointer is required, then we need to store the alloca'd result |
| 2257 | // in Dest. |
| 2258 | Operand *SubAmountRF = |
| 2259 | legalize(Ctx->getConstantInt32(Value), Legal_Reg | Legal_Flex); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 2260 | Sandboxer(this).sub_sp(SubAmountRF); |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2261 | } else { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 2262 | // Non-constant sizes need to be adjusted to the next highest multiple of |
| 2263 | // the required alignment at runtime. |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 2264 | TotalSize = legalize(TotalSize, Legal_Reg | Legal_Flex); |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2265 | Variable *T = makeReg(IceType_i32); |
| 2266 | _mov(T, TotalSize); |
| 2267 | Operand *AddAmount = legalize(Ctx->getConstantInt32(Alignment - 1)); |
| 2268 | _add(T, T, AddAmount); |
| 2269 | alignRegisterPow2(T, Alignment); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 2270 | Sandboxer(this).sub_sp(T); |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 2271 | } |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2272 | |
| 2273 | // Adds back a few bytes to SP to account for the out args area. |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 2274 | Variable *T = SP; |
| 2275 | if (MaxOutArgsSizeBytes != 0) { |
| 2276 | T = makeReg(getPointerType()); |
| 2277 | Operand *OutArgsSizeRF = legalize( |
| 2278 | Ctx->getConstantInt32(MaxOutArgsSizeBytes), Legal_Reg | Legal_Flex); |
| 2279 | _add(T, SP, OutArgsSizeRF); |
| 2280 | } |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 2281 | |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 2282 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 2283 | } |
| 2284 | |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2285 | void TargetARM32::div0Check(Type Ty, Operand *SrcLo, Operand *SrcHi) { |
| 2286 | if (isGuaranteedNonzeroInt(SrcLo) || isGuaranteedNonzeroInt(SrcHi)) |
| 2287 | return; |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 2288 | Variable *SrcLoReg = legalizeToReg(SrcLo); |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2289 | switch (Ty) { |
| 2290 | default: |
Eric Holk | cfc2553 | 2016-02-09 17:47:58 -0800 | [diff] [blame] | 2291 | llvm_unreachable( |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 2292 | ("Unexpected type in div0Check: " + typeStdString(Ty)).c_str()); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2293 | case IceType_i8: |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2294 | case IceType_i16: { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2295 | Operand *ShAmtImm = shAmtImm(32 - getScalarIntBitWidth(Ty)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2296 | Variable *T = makeReg(IceType_i32); |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2297 | _lsls(T, SrcLoReg, ShAmtImm); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 2298 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2299 | } break; |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2300 | case IceType_i32: { |
| 2301 | _tst(SrcLoReg, SrcLoReg); |
| 2302 | break; |
| 2303 | } |
| 2304 | case IceType_i64: { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2305 | Variable *T = makeReg(IceType_i32); |
| 2306 | _orrs(T, SrcLoReg, legalize(SrcHi, Legal_Reg | Legal_Flex)); |
| 2307 | // T isn't going to be used, but we need the side-effect of setting flags |
| 2308 | // from this operation. |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 2309 | Context.insert<InstFakeUse>(T); |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2310 | } |
| 2311 | } |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 2312 | auto *Label = InstARM32Label::create(Func, this); |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2313 | _br(Label, CondARM32::NE); |
| 2314 | _trap(); |
| 2315 | Context.insert(Label); |
| 2316 | } |
| 2317 | |
| 2318 | void TargetARM32::lowerIDivRem(Variable *Dest, Variable *T, Variable *Src0R, |
| 2319 | Operand *Src1, ExtInstr ExtFunc, |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 2320 | DivInstr DivFunc, bool IsRemainder) { |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2321 | div0Check(Dest->getType(), Src1, nullptr); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 2322 | Variable *Src1R = legalizeToReg(Src1); |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2323 | Variable *T0R = Src0R; |
| 2324 | Variable *T1R = Src1R; |
| 2325 | if (Dest->getType() != IceType_i32) { |
| 2326 | T0R = makeReg(IceType_i32); |
| 2327 | (this->*ExtFunc)(T0R, Src0R, CondARM32::AL); |
| 2328 | T1R = makeReg(IceType_i32); |
| 2329 | (this->*ExtFunc)(T1R, Src1R, CondARM32::AL); |
| 2330 | } |
| 2331 | if (hasCPUFeature(TargetARM32Features::HWDivArm)) { |
| 2332 | (this->*DivFunc)(T, T0R, T1R, CondARM32::AL); |
| 2333 | if (IsRemainder) { |
| 2334 | Variable *T2 = makeReg(IceType_i32); |
| 2335 | _mls(T2, T, T1R, T0R); |
| 2336 | T = T2; |
| 2337 | } |
| 2338 | _mov(Dest, T); |
| 2339 | } else { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 2340 | llvm::report_fatal_error("div should have already been turned into a call"); |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2341 | } |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 2342 | } |
| 2343 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 2344 | TargetARM32::SafeBoolChain |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2345 | TargetARM32::lowerInt1Arithmetic(const InstArithmetic *Instr) { |
| 2346 | Variable *Dest = Instr->getDest(); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 2347 | assert(Dest->getType() == IceType_i1); |
| 2348 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2349 | // So folding didn't work for Instr. Not a problem: We just need to |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 2350 | // materialize the Sources, and perform the operation. We create regular |
| 2351 | // Variables (and not infinite-weight ones) because this call might recurse a |
| 2352 | // lot, and we might end up with tons of infinite weight temporaries. |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2353 | assert(Instr->getSrcSize() == 2); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 2354 | Variable *Src0 = Func->makeVariable(IceType_i1); |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2355 | SafeBoolChain Src0Safe = lowerInt1(Src0, Instr->getSrc(0)); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 2356 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2357 | Operand *Src1 = Instr->getSrc(1); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 2358 | SafeBoolChain Src1Safe = SBC_Yes; |
| 2359 | |
| 2360 | if (!llvm::isa<Constant>(Src1)) { |
| 2361 | Variable *Src1V = Func->makeVariable(IceType_i1); |
| 2362 | Src1Safe = lowerInt1(Src1V, Src1); |
| 2363 | Src1 = Src1V; |
| 2364 | } |
| 2365 | |
| 2366 | Variable *T = makeReg(IceType_i1); |
| 2367 | Src0 = legalizeToReg(Src0); |
| 2368 | Operand *Src1RF = legalize(Src1, Legal_Reg | Legal_Flex); |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 2369 | switch (Instr->getOp()) { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 2370 | default: |
| 2371 | // If this Unreachable is ever executed, add the offending operation to |
| 2372 | // the list of valid consumers. |
| 2373 | llvm::report_fatal_error("Unhandled i1 Op"); |
| 2374 | case InstArithmetic::And: |
| 2375 | _and(T, Src0, Src1RF); |
| 2376 | break; |
| 2377 | case InstArithmetic::Or: |
| 2378 | _orr(T, Src0, Src1RF); |
| 2379 | break; |
| 2380 | case InstArithmetic::Xor: |
| 2381 | _eor(T, Src0, Src1RF); |
| 2382 | break; |
| 2383 | } |
| 2384 | _mov(Dest, T); |
| 2385 | return Src0Safe == SBC_Yes && Src1Safe == SBC_Yes ? SBC_Yes : SBC_No; |
| 2386 | } |
| 2387 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2388 | namespace { |
| 2389 | // NumericOperands is used during arithmetic/icmp lowering for constant folding. |
| 2390 | // It holds the two sources operands, and maintains some state as to whether one |
| 2391 | // of them is a constant. If one of the operands is a constant, then it will be |
| 2392 | // be stored as the operation's second source, with a bit indicating whether the |
| 2393 | // operands were swapped. |
| 2394 | // |
| 2395 | // The class is split into a base class with operand type-independent methods, |
| 2396 | // and a derived, templated class, for each type of operand we want to fold |
| 2397 | // constants for: |
| 2398 | // |
| 2399 | // NumericOperandsBase --> NumericOperands<ConstantFloat> |
| 2400 | // --> NumericOperands<ConstantDouble> |
| 2401 | // --> NumericOperands<ConstantInt32> |
| 2402 | // |
| 2403 | // NumericOperands<ConstantInt32> also exposes helper methods for emitting |
| 2404 | // inverted/negated immediates. |
| 2405 | class NumericOperandsBase { |
| 2406 | NumericOperandsBase() = delete; |
| 2407 | NumericOperandsBase(const NumericOperandsBase &) = delete; |
| 2408 | NumericOperandsBase &operator=(const NumericOperandsBase &) = delete; |
| 2409 | |
| 2410 | public: |
| 2411 | NumericOperandsBase(Operand *S0, Operand *S1) |
| 2412 | : Src0(NonConstOperand(S0, S1)), Src1(ConstOperand(S0, S1)), |
| 2413 | Swapped(Src0 == S1 && S0 != S1) { |
| 2414 | assert(Src0 != nullptr); |
| 2415 | assert(Src1 != nullptr); |
| 2416 | assert(Src0 != Src1 || S0 == S1); |
| 2417 | } |
| 2418 | |
| 2419 | bool hasConstOperand() const { |
| 2420 | return llvm::isa<Constant>(Src1) && !llvm::isa<ConstantRelocatable>(Src1); |
| 2421 | } |
| 2422 | |
| 2423 | bool swappedOperands() const { return Swapped; } |
| 2424 | |
| 2425 | Variable *src0R(TargetARM32 *Target) const { |
| 2426 | return legalizeToReg(Target, Src0); |
| 2427 | } |
| 2428 | |
| 2429 | Variable *unswappedSrc0R(TargetARM32 *Target) const { |
| 2430 | return legalizeToReg(Target, Swapped ? Src1 : Src0); |
| 2431 | } |
| 2432 | |
| 2433 | Operand *src1RF(TargetARM32 *Target) const { |
| 2434 | return legalizeToRegOrFlex(Target, Src1); |
| 2435 | } |
| 2436 | |
| 2437 | Variable *unswappedSrc1R(TargetARM32 *Target) const { |
| 2438 | return legalizeToReg(Target, Swapped ? Src0 : Src1); |
| 2439 | } |
| 2440 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2441 | protected: |
| 2442 | Operand *const Src0; |
| 2443 | Operand *const Src1; |
| 2444 | const bool Swapped; |
| 2445 | |
| 2446 | static Variable *legalizeToReg(TargetARM32 *Target, Operand *Src) { |
| 2447 | return Target->legalizeToReg(Src); |
| 2448 | } |
| 2449 | |
| 2450 | static Operand *legalizeToRegOrFlex(TargetARM32 *Target, Operand *Src) { |
| 2451 | return Target->legalize(Src, |
| 2452 | TargetARM32::Legal_Reg | TargetARM32::Legal_Flex); |
| 2453 | } |
| 2454 | |
| 2455 | private: |
| 2456 | static Operand *NonConstOperand(Operand *S0, Operand *S1) { |
| 2457 | if (!llvm::isa<Constant>(S0)) |
| 2458 | return S0; |
| 2459 | if (!llvm::isa<Constant>(S1)) |
| 2460 | return S1; |
| 2461 | if (llvm::isa<ConstantRelocatable>(S1) && |
| 2462 | !llvm::isa<ConstantRelocatable>(S0)) |
| 2463 | return S1; |
| 2464 | return S0; |
| 2465 | } |
| 2466 | |
| 2467 | static Operand *ConstOperand(Operand *S0, Operand *S1) { |
| 2468 | if (!llvm::isa<Constant>(S0)) |
| 2469 | return S1; |
| 2470 | if (!llvm::isa<Constant>(S1)) |
| 2471 | return S0; |
| 2472 | if (llvm::isa<ConstantRelocatable>(S1) && |
| 2473 | !llvm::isa<ConstantRelocatable>(S0)) |
| 2474 | return S0; |
| 2475 | return S1; |
| 2476 | } |
| 2477 | }; |
| 2478 | |
| 2479 | template <typename C> class NumericOperands : public NumericOperandsBase { |
| 2480 | NumericOperands() = delete; |
| 2481 | NumericOperands(const NumericOperands &) = delete; |
| 2482 | NumericOperands &operator=(const NumericOperands &) = delete; |
| 2483 | |
| 2484 | public: |
| 2485 | NumericOperands(Operand *S0, Operand *S1) : NumericOperandsBase(S0, S1) { |
| 2486 | assert(!hasConstOperand() || llvm::isa<C>(this->Src1)); |
| 2487 | } |
| 2488 | |
| 2489 | typename C::PrimType getConstantValue() const { |
| 2490 | return llvm::cast<C>(Src1)->getValue(); |
| 2491 | } |
| 2492 | }; |
| 2493 | |
| 2494 | using FloatOperands = NumericOperands<ConstantFloat>; |
| 2495 | using DoubleOperands = NumericOperands<ConstantDouble>; |
| 2496 | |
| 2497 | class Int32Operands : public NumericOperands<ConstantInteger32> { |
| 2498 | Int32Operands() = delete; |
| 2499 | Int32Operands(const Int32Operands &) = delete; |
| 2500 | Int32Operands &operator=(const Int32Operands &) = delete; |
| 2501 | |
| 2502 | public: |
| 2503 | Int32Operands(Operand *S0, Operand *S1) : NumericOperands(S0, S1) {} |
| 2504 | |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2505 | Operand *unswappedSrc1RShAmtImm(TargetARM32 *Target) const { |
| 2506 | if (!swappedOperands() && hasConstOperand()) { |
| 2507 | return Target->shAmtImm(getConstantValue() & 0x1F); |
| 2508 | } |
| 2509 | return legalizeToReg(Target, Swapped ? Src0 : Src1); |
| 2510 | } |
| 2511 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2512 | bool immediateIsFlexEncodable() const { |
| 2513 | uint32_t Rotate, Imm8; |
| 2514 | return OperandARM32FlexImm::canHoldImm(getConstantValue(), &Rotate, &Imm8); |
| 2515 | } |
| 2516 | |
| 2517 | bool negatedImmediateIsFlexEncodable() const { |
| 2518 | uint32_t Rotate, Imm8; |
| 2519 | return OperandARM32FlexImm::canHoldImm( |
| 2520 | -static_cast<int32_t>(getConstantValue()), &Rotate, &Imm8); |
| 2521 | } |
| 2522 | |
| 2523 | Operand *negatedSrc1F(TargetARM32 *Target) const { |
| 2524 | return legalizeToRegOrFlex(Target, |
| 2525 | Target->getCtx()->getConstantInt32( |
| 2526 | -static_cast<int32_t>(getConstantValue()))); |
| 2527 | } |
| 2528 | |
| 2529 | bool invertedImmediateIsFlexEncodable() const { |
| 2530 | uint32_t Rotate, Imm8; |
| 2531 | return OperandARM32FlexImm::canHoldImm( |
| 2532 | ~static_cast<uint32_t>(getConstantValue()), &Rotate, &Imm8); |
| 2533 | } |
| 2534 | |
| 2535 | Operand *invertedSrc1F(TargetARM32 *Target) const { |
| 2536 | return legalizeToRegOrFlex(Target, |
| 2537 | Target->getCtx()->getConstantInt32( |
| 2538 | ~static_cast<uint32_t>(getConstantValue()))); |
| 2539 | } |
| 2540 | }; |
| 2541 | } // end of anonymous namespace |
| 2542 | |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 2543 | void TargetARM32::preambleDivRem(const InstCall *Instr) { |
| 2544 | Operand *Src1 = Instr->getArg(1); |
| 2545 | |
| 2546 | switch (Src1->getType()) { |
| 2547 | default: |
| 2548 | llvm::report_fatal_error("Invalid type for idiv."); |
| 2549 | case IceType_i64: { |
| 2550 | if (auto *C = llvm::dyn_cast<ConstantInteger64>(Src1)) { |
| 2551 | if (C->getValue() == 0) { |
| 2552 | _trap(); |
| 2553 | return; |
| 2554 | } |
| 2555 | } |
| 2556 | div0Check(IceType_i64, loOperand(Src1), hiOperand(Src1)); |
| 2557 | return; |
| 2558 | } |
| 2559 | case IceType_i32: { |
| 2560 | // Src0 and Src1 have already been appropriately extended to an i32, so we |
| 2561 | // don't check for i8 and i16. |
| 2562 | if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { |
| 2563 | if (C->getValue() == 0) { |
| 2564 | _trap(); |
| 2565 | return; |
| 2566 | } |
| 2567 | } |
| 2568 | div0Check(IceType_i32, Src1, nullptr); |
| 2569 | return; |
| 2570 | } |
| 2571 | } |
| 2572 | } |
| 2573 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2574 | void TargetARM32::lowerInt64Arithmetic(InstArithmetic::OpKind Op, |
| 2575 | Variable *Dest, Operand *Src0, |
| 2576 | Operand *Src1) { |
| 2577 | Int32Operands SrcsLo(loOperand(Src0), loOperand(Src1)); |
| 2578 | Int32Operands SrcsHi(hiOperand(Src0), hiOperand(Src1)); |
| 2579 | assert(SrcsLo.swappedOperands() == SrcsHi.swappedOperands()); |
| 2580 | assert(SrcsLo.hasConstOperand() == SrcsHi.hasConstOperand()); |
| 2581 | |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 2582 | auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 2583 | auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2584 | Variable *T_Lo = makeReg(DestLo->getType()); |
| 2585 | Variable *T_Hi = makeReg(DestHi->getType()); |
| 2586 | |
| 2587 | switch (Op) { |
| 2588 | case InstArithmetic::_num: |
| 2589 | llvm::report_fatal_error("Unknown arithmetic operator"); |
| 2590 | return; |
| 2591 | case InstArithmetic::Add: { |
| 2592 | Variable *Src0LoR = SrcsLo.src0R(this); |
| 2593 | Operand *Src1LoRF = SrcsLo.src1RF(this); |
| 2594 | Variable *Src0HiR = SrcsHi.src0R(this); |
| 2595 | Operand *Src1HiRF = SrcsHi.src1RF(this); |
| 2596 | _adds(T_Lo, Src0LoR, Src1LoRF); |
| 2597 | _mov(DestLo, T_Lo); |
| 2598 | _adc(T_Hi, Src0HiR, Src1HiRF); |
| 2599 | _mov(DestHi, T_Hi); |
| 2600 | return; |
| 2601 | } |
| 2602 | case InstArithmetic::And: { |
| 2603 | Variable *Src0LoR = SrcsLo.src0R(this); |
| 2604 | Operand *Src1LoRF = SrcsLo.src1RF(this); |
| 2605 | Variable *Src0HiR = SrcsHi.src0R(this); |
| 2606 | Operand *Src1HiRF = SrcsHi.src1RF(this); |
| 2607 | _and(T_Lo, Src0LoR, Src1LoRF); |
| 2608 | _mov(DestLo, T_Lo); |
| 2609 | _and(T_Hi, Src0HiR, Src1HiRF); |
| 2610 | _mov(DestHi, T_Hi); |
| 2611 | return; |
| 2612 | } |
| 2613 | case InstArithmetic::Or: { |
| 2614 | Variable *Src0LoR = SrcsLo.src0R(this); |
| 2615 | Operand *Src1LoRF = SrcsLo.src1RF(this); |
| 2616 | Variable *Src0HiR = SrcsHi.src0R(this); |
| 2617 | Operand *Src1HiRF = SrcsHi.src1RF(this); |
| 2618 | _orr(T_Lo, Src0LoR, Src1LoRF); |
| 2619 | _mov(DestLo, T_Lo); |
| 2620 | _orr(T_Hi, Src0HiR, Src1HiRF); |
| 2621 | _mov(DestHi, T_Hi); |
| 2622 | return; |
| 2623 | } |
| 2624 | case InstArithmetic::Xor: { |
| 2625 | Variable *Src0LoR = SrcsLo.src0R(this); |
| 2626 | Operand *Src1LoRF = SrcsLo.src1RF(this); |
| 2627 | Variable *Src0HiR = SrcsHi.src0R(this); |
| 2628 | Operand *Src1HiRF = SrcsHi.src1RF(this); |
| 2629 | _eor(T_Lo, Src0LoR, Src1LoRF); |
| 2630 | _mov(DestLo, T_Lo); |
| 2631 | _eor(T_Hi, Src0HiR, Src1HiRF); |
| 2632 | _mov(DestHi, T_Hi); |
| 2633 | return; |
| 2634 | } |
| 2635 | case InstArithmetic::Sub: { |
| 2636 | Variable *Src0LoR = SrcsLo.src0R(this); |
| 2637 | Operand *Src1LoRF = SrcsLo.src1RF(this); |
| 2638 | Variable *Src0HiR = SrcsHi.src0R(this); |
| 2639 | Operand *Src1HiRF = SrcsHi.src1RF(this); |
| 2640 | if (SrcsLo.swappedOperands()) { |
| 2641 | _rsbs(T_Lo, Src0LoR, Src1LoRF); |
| 2642 | _mov(DestLo, T_Lo); |
| 2643 | _rsc(T_Hi, Src0HiR, Src1HiRF); |
| 2644 | _mov(DestHi, T_Hi); |
| 2645 | } else { |
| 2646 | _subs(T_Lo, Src0LoR, Src1LoRF); |
| 2647 | _mov(DestLo, T_Lo); |
| 2648 | _sbc(T_Hi, Src0HiR, Src1HiRF); |
| 2649 | _mov(DestHi, T_Hi); |
| 2650 | } |
| 2651 | return; |
| 2652 | } |
| 2653 | case InstArithmetic::Mul: { |
| 2654 | // GCC 4.8 does: |
| 2655 | // a=b*c ==> |
| 2656 | // t_acc =(mul) (b.lo * c.hi) |
| 2657 | // t_acc =(mla) (c.lo * b.hi) + t_acc |
| 2658 | // t.hi,t.lo =(umull) b.lo * c.lo |
| 2659 | // t.hi += t_acc |
| 2660 | // a.lo = t.lo |
| 2661 | // a.hi = t.hi |
| 2662 | // |
| 2663 | // LLVM does: |
| 2664 | // t.hi,t.lo =(umull) b.lo * c.lo |
| 2665 | // t.hi =(mla) (b.lo * c.hi) + t.hi |
| 2666 | // t.hi =(mla) (b.hi * c.lo) + t.hi |
| 2667 | // a.lo = t.lo |
| 2668 | // a.hi = t.hi |
| 2669 | // |
| 2670 | // LLVM's lowering has fewer instructions, but more register pressure: |
| 2671 | // t.lo is live from beginning to end, while GCC delays the two-dest |
| 2672 | // instruction till the end, and kills c.hi immediately. |
| 2673 | Variable *T_Acc = makeReg(IceType_i32); |
| 2674 | Variable *T_Acc1 = makeReg(IceType_i32); |
| 2675 | Variable *T_Hi1 = makeReg(IceType_i32); |
| 2676 | Variable *Src0RLo = SrcsLo.unswappedSrc0R(this); |
| 2677 | Variable *Src0RHi = SrcsHi.unswappedSrc0R(this); |
| 2678 | Variable *Src1RLo = SrcsLo.unswappedSrc1R(this); |
| 2679 | Variable *Src1RHi = SrcsHi.unswappedSrc1R(this); |
| 2680 | _mul(T_Acc, Src0RLo, Src1RHi); |
| 2681 | _mla(T_Acc1, Src1RLo, Src0RHi, T_Acc); |
| 2682 | _umull(T_Lo, T_Hi1, Src0RLo, Src1RLo); |
| 2683 | _add(T_Hi, T_Hi1, T_Acc1); |
| 2684 | _mov(DestLo, T_Lo); |
| 2685 | _mov(DestHi, T_Hi); |
| 2686 | return; |
| 2687 | } |
| 2688 | case InstArithmetic::Shl: { |
| 2689 | if (!SrcsLo.swappedOperands() && SrcsLo.hasConstOperand()) { |
| 2690 | Variable *Src0RLo = SrcsLo.src0R(this); |
| 2691 | // Truncating the ShAmt to [0, 63] because that's what ARM does anyway. |
| 2692 | const int32_t ShAmtImm = SrcsLo.getConstantValue() & 0x3F; |
| 2693 | if (ShAmtImm == 0) { |
| 2694 | _mov(DestLo, Src0RLo); |
| 2695 | _mov(DestHi, SrcsHi.src0R(this)); |
| 2696 | return; |
| 2697 | } |
| 2698 | |
| 2699 | if (ShAmtImm >= 32) { |
| 2700 | if (ShAmtImm == 32) { |
| 2701 | _mov(DestHi, Src0RLo); |
| 2702 | } else { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2703 | Operand *ShAmtOp = shAmtImm(ShAmtImm - 32); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2704 | _lsl(T_Hi, Src0RLo, ShAmtOp); |
| 2705 | _mov(DestHi, T_Hi); |
| 2706 | } |
| 2707 | |
| 2708 | Operand *_0 = |
| 2709 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
| 2710 | _mov(T_Lo, _0); |
| 2711 | _mov(DestLo, T_Lo); |
| 2712 | return; |
| 2713 | } |
| 2714 | |
| 2715 | Variable *Src0RHi = SrcsHi.src0R(this); |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2716 | Operand *ShAmtOp = shAmtImm(ShAmtImm); |
| 2717 | Operand *ComplShAmtOp = shAmtImm(32 - ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2718 | _lsl(T_Hi, Src0RHi, ShAmtOp); |
| 2719 | _orr(T_Hi, T_Hi, |
| 2720 | OperandARM32FlexReg::create(Func, IceType_i32, Src0RLo, |
| 2721 | OperandARM32::LSR, ComplShAmtOp)); |
| 2722 | _mov(DestHi, T_Hi); |
| 2723 | |
| 2724 | _lsl(T_Lo, Src0RLo, ShAmtOp); |
| 2725 | _mov(DestLo, T_Lo); |
| 2726 | return; |
| 2727 | } |
| 2728 | |
| 2729 | // a=b<<c ==> |
| 2730 | // pnacl-llc does: |
| 2731 | // mov t_b.lo, b.lo |
| 2732 | // mov t_b.hi, b.hi |
| 2733 | // mov t_c.lo, c.lo |
| 2734 | // rsb T0, t_c.lo, #32 |
| 2735 | // lsr T1, t_b.lo, T0 |
| 2736 | // orr t_a.hi, T1, t_b.hi, lsl t_c.lo |
| 2737 | // sub T2, t_c.lo, #32 |
| 2738 | // cmp T2, #0 |
| 2739 | // lslge t_a.hi, t_b.lo, T2 |
| 2740 | // lsl t_a.lo, t_b.lo, t_c.lo |
| 2741 | // mov a.lo, t_a.lo |
| 2742 | // mov a.hi, t_a.hi |
| 2743 | // |
| 2744 | // GCC 4.8 does: |
| 2745 | // sub t_c1, c.lo, #32 |
| 2746 | // lsl t_hi, b.hi, c.lo |
| 2747 | // orr t_hi, t_hi, b.lo, lsl t_c1 |
| 2748 | // rsb t_c2, c.lo, #32 |
| 2749 | // orr t_hi, t_hi, b.lo, lsr t_c2 |
| 2750 | // lsl t_lo, b.lo, c.lo |
| 2751 | // a.lo = t_lo |
| 2752 | // a.hi = t_hi |
| 2753 | // |
| 2754 | // These are incompatible, therefore we mimic pnacl-llc. |
| 2755 | // Can be strength-reduced for constant-shifts, but we don't do that for |
| 2756 | // now. |
| 2757 | // Given the sub/rsb T_C, C.lo, #32, one of the T_C will be negative. On |
| 2758 | // ARM, shifts only take the lower 8 bits of the shift register, and |
| 2759 | // saturate to the range 0-32, so the negative value will saturate to 32. |
| 2760 | Operand *_32 = legalize(Ctx->getConstantInt32(32), Legal_Reg | Legal_Flex); |
| 2761 | Operand *_0 = |
| 2762 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
| 2763 | Variable *T0 = makeReg(IceType_i32); |
| 2764 | Variable *T1 = makeReg(IceType_i32); |
| 2765 | Variable *T2 = makeReg(IceType_i32); |
| 2766 | Variable *TA_Hi = makeReg(IceType_i32); |
| 2767 | Variable *TA_Lo = makeReg(IceType_i32); |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 2768 | Variable *Src0RLo = SrcsLo.unswappedSrc0R(this); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2769 | Variable *Src0RHi = SrcsHi.unswappedSrc0R(this); |
| 2770 | Variable *Src1RLo = SrcsLo.unswappedSrc1R(this); |
| 2771 | _rsb(T0, Src1RLo, _32); |
| 2772 | _lsr(T1, Src0RLo, T0); |
| 2773 | _orr(TA_Hi, T1, OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi, |
| 2774 | OperandARM32::LSL, Src1RLo)); |
| 2775 | _sub(T2, Src1RLo, _32); |
| 2776 | _cmp(T2, _0); |
| 2777 | _lsl(TA_Hi, Src0RLo, T2, CondARM32::GE); |
| 2778 | _set_dest_redefined(); |
| 2779 | _lsl(TA_Lo, Src0RLo, Src1RLo); |
| 2780 | _mov(DestLo, TA_Lo); |
| 2781 | _mov(DestHi, TA_Hi); |
| 2782 | return; |
| 2783 | } |
| 2784 | case InstArithmetic::Lshr: |
| 2785 | case InstArithmetic::Ashr: { |
| 2786 | const bool ASR = Op == InstArithmetic::Ashr; |
| 2787 | if (!SrcsLo.swappedOperands() && SrcsLo.hasConstOperand()) { |
| 2788 | Variable *Src0RHi = SrcsHi.src0R(this); |
| 2789 | // Truncating the ShAmt to [0, 63] because that's what ARM does anyway. |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2790 | const int32_t ShAmt = SrcsLo.getConstantValue() & 0x3F; |
| 2791 | if (ShAmt == 0) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2792 | _mov(DestHi, Src0RHi); |
| 2793 | _mov(DestLo, SrcsLo.src0R(this)); |
| 2794 | return; |
| 2795 | } |
| 2796 | |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2797 | if (ShAmt >= 32) { |
| 2798 | if (ShAmt == 32) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2799 | _mov(DestLo, Src0RHi); |
| 2800 | } else { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2801 | Operand *ShAmtImm = shAmtImm(ShAmt - 32); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2802 | if (ASR) { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2803 | _asr(T_Lo, Src0RHi, ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2804 | } else { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2805 | _lsr(T_Lo, Src0RHi, ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2806 | } |
| 2807 | _mov(DestLo, T_Lo); |
| 2808 | } |
| 2809 | |
| 2810 | if (ASR) { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2811 | Operand *_31 = shAmtImm(31); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2812 | _asr(T_Hi, Src0RHi, _31); |
| 2813 | } else { |
| 2814 | Operand *_0 = legalize(Ctx->getConstantZero(IceType_i32), |
| 2815 | Legal_Reg | Legal_Flex); |
| 2816 | _mov(T_Hi, _0); |
| 2817 | } |
| 2818 | _mov(DestHi, T_Hi); |
| 2819 | return; |
| 2820 | } |
| 2821 | |
| 2822 | Variable *Src0RLo = SrcsLo.src0R(this); |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2823 | Operand *ShAmtImm = shAmtImm(ShAmt); |
| 2824 | Operand *ComplShAmtImm = shAmtImm(32 - ShAmt); |
| 2825 | _lsr(T_Lo, Src0RLo, ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2826 | _orr(T_Lo, T_Lo, |
| 2827 | OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi, |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2828 | OperandARM32::LSL, ComplShAmtImm)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2829 | _mov(DestLo, T_Lo); |
| 2830 | |
| 2831 | if (ASR) { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2832 | _asr(T_Hi, Src0RHi, ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2833 | } else { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 2834 | _lsr(T_Hi, Src0RHi, ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 2835 | } |
| 2836 | _mov(DestHi, T_Hi); |
| 2837 | return; |
| 2838 | } |
| 2839 | |
| 2840 | // a=b>>c |
| 2841 | // pnacl-llc does: |
| 2842 | // mov t_b.lo, b.lo |
| 2843 | // mov t_b.hi, b.hi |
| 2844 | // mov t_c.lo, c.lo |
| 2845 | // lsr T0, t_b.lo, t_c.lo |
| 2846 | // rsb T1, t_c.lo, #32 |
| 2847 | // orr t_a.lo, T0, t_b.hi, lsl T1 |
| 2848 | // sub T2, t_c.lo, #32 |
| 2849 | // cmp T2, #0 |
| 2850 | // [al]srge t_a.lo, t_b.hi, T2 |
| 2851 | // [al]sr t_a.hi, t_b.hi, t_c.lo |
| 2852 | // mov a.lo, t_a.lo |
| 2853 | // mov a.hi, t_a.hi |
| 2854 | // |
| 2855 | // GCC 4.8 does (lsr): |
| 2856 | // rsb t_c1, c.lo, #32 |
| 2857 | // lsr t_lo, b.lo, c.lo |
| 2858 | // orr t_lo, t_lo, b.hi, lsl t_c1 |
| 2859 | // sub t_c2, c.lo, #32 |
| 2860 | // orr t_lo, t_lo, b.hi, lsr t_c2 |
| 2861 | // lsr t_hi, b.hi, c.lo |
| 2862 | // mov a.lo, t_lo |
| 2863 | // mov a.hi, t_hi |
| 2864 | // |
| 2865 | // These are incompatible, therefore we mimic pnacl-llc. |
| 2866 | Operand *_32 = legalize(Ctx->getConstantInt32(32), Legal_Reg | Legal_Flex); |
| 2867 | Operand *_0 = |
| 2868 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
| 2869 | Variable *T0 = makeReg(IceType_i32); |
| 2870 | Variable *T1 = makeReg(IceType_i32); |
| 2871 | Variable *T2 = makeReg(IceType_i32); |
| 2872 | Variable *TA_Lo = makeReg(IceType_i32); |
| 2873 | Variable *TA_Hi = makeReg(IceType_i32); |
| 2874 | Variable *Src0RLo = SrcsLo.unswappedSrc0R(this); |
| 2875 | Variable *Src0RHi = SrcsHi.unswappedSrc0R(this); |
| 2876 | Variable *Src1RLo = SrcsLo.unswappedSrc1R(this); |
| 2877 | _lsr(T0, Src0RLo, Src1RLo); |
| 2878 | _rsb(T1, Src1RLo, _32); |
| 2879 | _orr(TA_Lo, T0, OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi, |
| 2880 | OperandARM32::LSL, T1)); |
| 2881 | _sub(T2, Src1RLo, _32); |
| 2882 | _cmp(T2, _0); |
| 2883 | if (ASR) { |
| 2884 | _asr(TA_Lo, Src0RHi, T2, CondARM32::GE); |
| 2885 | _set_dest_redefined(); |
| 2886 | _asr(TA_Hi, Src0RHi, Src1RLo); |
| 2887 | } else { |
| 2888 | _lsr(TA_Lo, Src0RHi, T2, CondARM32::GE); |
| 2889 | _set_dest_redefined(); |
| 2890 | _lsr(TA_Hi, Src0RHi, Src1RLo); |
| 2891 | } |
| 2892 | _mov(DestLo, TA_Lo); |
| 2893 | _mov(DestHi, TA_Hi); |
| 2894 | return; |
| 2895 | } |
| 2896 | case InstArithmetic::Fadd: |
| 2897 | case InstArithmetic::Fsub: |
| 2898 | case InstArithmetic::Fmul: |
| 2899 | case InstArithmetic::Fdiv: |
| 2900 | case InstArithmetic::Frem: |
| 2901 | llvm::report_fatal_error("FP instruction with i64 type"); |
| 2902 | return; |
| 2903 | case InstArithmetic::Udiv: |
| 2904 | case InstArithmetic::Sdiv: |
| 2905 | case InstArithmetic::Urem: |
| 2906 | case InstArithmetic::Srem: |
| 2907 | llvm::report_fatal_error("Call-helper-involved instruction for i64 type " |
| 2908 | "should have already been handled before"); |
| 2909 | return; |
| 2910 | } |
| 2911 | } |
| 2912 | |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 2913 | namespace { |
| 2914 | // StrengthReduction is a namespace with the strength reduction machinery. The |
| 2915 | // entry point is the StrengthReduction::tryToOptimize method. It returns true |
| 2916 | // if the optimization can be performed, and false otherwise. |
| 2917 | // |
| 2918 | // If the optimization can be performed, tryToOptimize sets its NumOperations |
| 2919 | // parameter to the number of shifts that are needed to perform the |
| 2920 | // multiplication; and it sets the Operations parameter with <ShAmt, AddOrSub> |
| 2921 | // tuples that describe how to materialize the multiplication. |
| 2922 | // |
| 2923 | // The algorithm finds contiguous 1s in the Multiplication source, and uses one |
| 2924 | // or two shifts to materialize it. A sequence of 1s, e.g., |
| 2925 | // |
| 2926 | // M N |
| 2927 | // ...00000000000011111...111110000000... |
| 2928 | // |
| 2929 | // is materializable with (1 << (M + 1)) - (1 << N): |
| 2930 | // |
| 2931 | // ...00000000000100000...000000000000... [1 << (M + 1)] |
| 2932 | // ...00000000000000000...000010000000... (-) [1 << N] |
| 2933 | // -------------------------------------- |
| 2934 | // ...00000000000011111...111110000000... |
| 2935 | // |
| 2936 | // And a single bit set, which is just a left shift. |
| 2937 | namespace StrengthReduction { |
| 2938 | enum AggregationOperation { |
| 2939 | AO_Invalid, |
| 2940 | AO_Add, |
| 2941 | AO_Sub, |
| 2942 | }; |
| 2943 | |
| 2944 | // AggregateElement is a glorified <ShAmt, AddOrSub> tuple. |
| 2945 | class AggregationElement { |
| 2946 | AggregationElement(const AggregationElement &) = delete; |
| 2947 | |
| 2948 | public: |
| 2949 | AggregationElement() = default; |
| 2950 | AggregationElement &operator=(const AggregationElement &) = default; |
| 2951 | AggregationElement(AggregationOperation Op, uint32_t ShAmt) |
| 2952 | : Op(Op), ShAmt(ShAmt) {} |
| 2953 | |
| 2954 | Operand *createShiftedOperand(Cfg *Func, Variable *OpR) const { |
| 2955 | assert(OpR->mustHaveReg()); |
| 2956 | if (ShAmt == 0) { |
| 2957 | return OpR; |
| 2958 | } |
| 2959 | return OperandARM32FlexReg::create( |
| 2960 | Func, IceType_i32, OpR, OperandARM32::LSL, |
| 2961 | OperandARM32ShAmtImm::create( |
| 2962 | Func, llvm::cast<ConstantInteger32>( |
| 2963 | Func->getContext()->getConstantInt32(ShAmt)))); |
| 2964 | } |
| 2965 | |
| 2966 | bool aggregateWithAdd() const { |
| 2967 | switch (Op) { |
| 2968 | case AO_Invalid: |
| 2969 | llvm::report_fatal_error("Invalid Strength Reduction Operations."); |
| 2970 | case AO_Add: |
| 2971 | return true; |
| 2972 | case AO_Sub: |
| 2973 | return false; |
| 2974 | } |
Jim Stichnoth | b0051df | 2016-01-13 11:39:15 -0800 | [diff] [blame] | 2975 | llvm_unreachable("(silence g++ warning)"); |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 2976 | } |
| 2977 | |
| 2978 | uint32_t shAmt() const { return ShAmt; } |
| 2979 | |
| 2980 | private: |
| 2981 | AggregationOperation Op = AO_Invalid; |
| 2982 | uint32_t ShAmt; |
| 2983 | }; |
| 2984 | |
| 2985 | // [RangeStart, RangeEnd] is a range of 1s in Src. |
| 2986 | template <std::size_t N> |
| 2987 | bool addOperations(uint32_t RangeStart, uint32_t RangeEnd, SizeT *NumOperations, |
| 2988 | std::array<AggregationElement, N> *Operations) { |
| 2989 | assert(*NumOperations < N); |
| 2990 | if (RangeStart == RangeEnd) { |
| 2991 | // Single bit set: |
| 2992 | // Src : 0...00010... |
| 2993 | // RangeStart : ^ |
| 2994 | // RangeEnd : ^ |
| 2995 | // NegSrc : 0...00001... |
| 2996 | (*Operations)[*NumOperations] = AggregationElement(AO_Add, RangeStart); |
| 2997 | ++(*NumOperations); |
| 2998 | return true; |
| 2999 | } |
| 3000 | |
| 3001 | // Sequence of 1s: (two operations required.) |
| 3002 | // Src : 0...00011...110... |
| 3003 | // RangeStart : ^ |
| 3004 | // RangeEnd : ^ |
| 3005 | // NegSrc : 0...00000...001... |
| 3006 | if (*NumOperations + 1 >= N) { |
| 3007 | return false; |
| 3008 | } |
| 3009 | (*Operations)[*NumOperations] = AggregationElement(AO_Add, RangeStart + 1); |
| 3010 | ++(*NumOperations); |
| 3011 | (*Operations)[*NumOperations] = AggregationElement(AO_Sub, RangeEnd); |
| 3012 | ++(*NumOperations); |
| 3013 | return true; |
| 3014 | } |
| 3015 | |
| 3016 | // tryToOptmize scans Src looking for sequences of 1s (including the unitary bit |
| 3017 | // 1 surrounded by zeroes. |
| 3018 | template <std::size_t N> |
| 3019 | bool tryToOptimize(uint32_t Src, SizeT *NumOperations, |
| 3020 | std::array<AggregationElement, N> *Operations) { |
| 3021 | constexpr uint32_t SrcSizeBits = sizeof(Src) * CHAR_BIT; |
| 3022 | uint32_t NegSrc = ~Src; |
| 3023 | |
| 3024 | *NumOperations = 0; |
| 3025 | while (Src != 0 && *NumOperations < N) { |
| 3026 | // Each step of the algorithm: |
| 3027 | // * finds L, the last bit set in Src; |
| 3028 | // * clears all the upper bits in NegSrc up to bit L; |
| 3029 | // * finds nL, the last bit set in NegSrc; |
| 3030 | // * clears all the upper bits in Src up to bit nL; |
| 3031 | // |
| 3032 | // if L == nL + 1, then a unitary 1 was found in Src. Otherwise, a sequence |
| 3033 | // of 1s starting at L, and ending at nL + 1, was found. |
| 3034 | const uint32_t SrcLastBitSet = llvm::findLastSet(Src); |
| 3035 | const uint32_t NegSrcClearMask = |
| 3036 | (SrcLastBitSet == 0) ? 0 |
| 3037 | : (0xFFFFFFFFu) >> (SrcSizeBits - SrcLastBitSet); |
| 3038 | NegSrc &= NegSrcClearMask; |
| 3039 | if (NegSrc == 0) { |
| 3040 | if (addOperations(SrcLastBitSet, 0, NumOperations, Operations)) { |
| 3041 | return true; |
| 3042 | } |
| 3043 | return false; |
| 3044 | } |
| 3045 | const uint32_t NegSrcLastBitSet = llvm::findLastSet(NegSrc); |
| 3046 | assert(NegSrcLastBitSet < SrcLastBitSet); |
| 3047 | const uint32_t SrcClearMask = |
| 3048 | (NegSrcLastBitSet == 0) ? 0 : (0xFFFFFFFFu) >> |
| 3049 | (SrcSizeBits - NegSrcLastBitSet); |
| 3050 | Src &= SrcClearMask; |
| 3051 | if (!addOperations(SrcLastBitSet, NegSrcLastBitSet + 1, NumOperations, |
| 3052 | Operations)) { |
| 3053 | return false; |
| 3054 | } |
| 3055 | } |
| 3056 | |
| 3057 | return Src == 0; |
| 3058 | } |
| 3059 | } // end of namespace StrengthReduction |
| 3060 | } // end of anonymous namespace |
| 3061 | |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3062 | void TargetARM32::lowerArithmetic(const InstArithmetic *Instr) { |
| 3063 | Variable *Dest = Instr->getDest(); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 3064 | |
| 3065 | if (Dest->isRematerializable()) { |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 3066 | Context.insert<InstFakeDef>(Dest); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 3067 | return; |
| 3068 | } |
| 3069 | |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 3070 | Type DestTy = Dest->getType(); |
| 3071 | if (DestTy == IceType_i1) { |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3072 | lowerInt1Arithmetic(Instr); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3073 | return; |
| 3074 | } |
| 3075 | |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3076 | Operand *Src0 = legalizeUndef(Instr->getSrc(0)); |
| 3077 | Operand *Src1 = legalizeUndef(Instr->getSrc(1)); |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 3078 | if (DestTy == IceType_i64) { |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3079 | lowerInt64Arithmetic(Instr->getOp(), Instr->getDest(), Src0, Src1); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3080 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3081 | } |
| 3082 | |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 3083 | if (isVectorType(DestTy)) { |
Eric Holk | 40c69b4 | 2016-01-26 10:10:39 -0800 | [diff] [blame] | 3084 | switch (Instr->getOp()) { |
| 3085 | default: |
| 3086 | UnimplementedLoweringError(this, Instr); |
| 3087 | return; |
| 3088 | // Explicitly whitelist vector instructions we have implemented/enabled. |
Eric Holk | 40c69b4 | 2016-01-26 10:10:39 -0800 | [diff] [blame] | 3089 | case InstArithmetic::Add: |
Eric Holk | b58170c | 2016-01-27 11:18:29 -0800 | [diff] [blame] | 3090 | case InstArithmetic::And: |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 3091 | case InstArithmetic::Ashr: |
| 3092 | case InstArithmetic::Fadd: |
Eric Holk | 029bed9 | 2016-01-28 13:38:43 -0800 | [diff] [blame] | 3093 | case InstArithmetic::Fmul: |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 3094 | case InstArithmetic::Fsub: |
| 3095 | case InstArithmetic::Lshr: |
Eric Holk | 029bed9 | 2016-01-28 13:38:43 -0800 | [diff] [blame] | 3096 | case InstArithmetic::Mul: |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 3097 | case InstArithmetic::Or: |
| 3098 | case InstArithmetic::Shl: |
| 3099 | case InstArithmetic::Sub: |
| 3100 | case InstArithmetic::Xor: |
Eric Holk | 40c69b4 | 2016-01-26 10:10:39 -0800 | [diff] [blame] | 3101 | break; |
| 3102 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3103 | } |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3104 | |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 3105 | Variable *T = makeReg(DestTy); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3106 | |
| 3107 | // * Handle div/rem separately. They require a non-legalized Src1 to inspect |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3108 | // whether or not Src1 is a non-zero constant. Once legalized it is more |
| 3109 | // difficult to determine (constant may be moved to a register). |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3110 | // * Handle floating point arithmetic separately: they require Src1 to be |
| 3111 | // legalized to a register. |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3112 | switch (Instr->getOp()) { |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3113 | default: |
| 3114 | break; |
| 3115 | case InstArithmetic::Udiv: { |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 3116 | constexpr bool NotRemainder = false; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3117 | Variable *Src0R = legalizeToReg(Src0); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3118 | lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_uxt, &TargetARM32::_udiv, |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3119 | NotRemainder); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3120 | return; |
| 3121 | } |
| 3122 | case InstArithmetic::Sdiv: { |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 3123 | constexpr bool NotRemainder = false; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3124 | Variable *Src0R = legalizeToReg(Src0); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3125 | lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_sxt, &TargetARM32::_sdiv, |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3126 | NotRemainder); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3127 | return; |
| 3128 | } |
| 3129 | case InstArithmetic::Urem: { |
| 3130 | constexpr bool IsRemainder = true; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3131 | Variable *Src0R = legalizeToReg(Src0); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3132 | lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_uxt, &TargetARM32::_udiv, |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3133 | IsRemainder); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3134 | return; |
| 3135 | } |
| 3136 | case InstArithmetic::Srem: { |
| 3137 | constexpr bool IsRemainder = true; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3138 | Variable *Src0R = legalizeToReg(Src0); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3139 | lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_sxt, &TargetARM32::_sdiv, |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3140 | IsRemainder); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3141 | return; |
| 3142 | } |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3143 | case InstArithmetic::Frem: { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3144 | if (!isScalarFloatingType(DestTy)) { |
| 3145 | llvm::report_fatal_error("Unexpected type when lowering frem."); |
| 3146 | } |
| 3147 | llvm::report_fatal_error("Frem should have already been lowered."); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3148 | } |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3149 | case InstArithmetic::Fadd: { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3150 | Variable *Src0R = legalizeToReg(Src0); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3151 | if (const Inst *Src1Producer = Computations.getProducerOf(Src1)) { |
| 3152 | Variable *Src1R = legalizeToReg(Src1Producer->getSrc(0)); |
| 3153 | Variable *Src2R = legalizeToReg(Src1Producer->getSrc(1)); |
| 3154 | _vmla(Src0R, Src1R, Src2R); |
| 3155 | _mov(Dest, Src0R); |
| 3156 | return; |
| 3157 | } |
| 3158 | |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3159 | Variable *Src1R = legalizeToReg(Src1); |
| 3160 | _vadd(T, Src0R, Src1R); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3161 | _mov(Dest, T); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3162 | return; |
| 3163 | } |
| 3164 | case InstArithmetic::Fsub: { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3165 | Variable *Src0R = legalizeToReg(Src0); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3166 | if (const Inst *Src1Producer = Computations.getProducerOf(Src1)) { |
| 3167 | Variable *Src1R = legalizeToReg(Src1Producer->getSrc(0)); |
| 3168 | Variable *Src2R = legalizeToReg(Src1Producer->getSrc(1)); |
| 3169 | _vmls(Src0R, Src1R, Src2R); |
| 3170 | _mov(Dest, Src0R); |
| 3171 | return; |
| 3172 | } |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3173 | Variable *Src1R = legalizeToReg(Src1); |
| 3174 | _vsub(T, Src0R, Src1R); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3175 | _mov(Dest, T); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3176 | return; |
| 3177 | } |
| 3178 | case InstArithmetic::Fmul: { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3179 | Variable *Src0R = legalizeToReg(Src0); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3180 | Variable *Src1R = legalizeToReg(Src1); |
| 3181 | _vmul(T, Src0R, Src1R); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3182 | _mov(Dest, T); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3183 | return; |
| 3184 | } |
| 3185 | case InstArithmetic::Fdiv: { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3186 | Variable *Src0R = legalizeToReg(Src0); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3187 | Variable *Src1R = legalizeToReg(Src1); |
| 3188 | _vdiv(T, Src0R, Src1R); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3189 | _mov(Dest, T); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3190 | return; |
| 3191 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3192 | } |
| 3193 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3194 | // Handle everything else here. |
| 3195 | Int32Operands Srcs(Src0, Src1); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3196 | switch (Instr->getOp()) { |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3197 | case InstArithmetic::_num: |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3198 | llvm::report_fatal_error("Unknown arithmetic operator"); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3199 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3200 | case InstArithmetic::Add: { |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3201 | if (const Inst *Src1Producer = Computations.getProducerOf(Src1)) { |
Eric Holk | 40c69b4 | 2016-01-26 10:10:39 -0800 | [diff] [blame] | 3202 | assert(!isVectorType(DestTy)); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3203 | Variable *Src0R = legalizeToReg(Src0); |
| 3204 | Variable *Src1R = legalizeToReg(Src1Producer->getSrc(0)); |
| 3205 | Variable *Src2R = legalizeToReg(Src1Producer->getSrc(1)); |
| 3206 | _mla(T, Src1R, Src2R, Src0R); |
| 3207 | _mov(Dest, T); |
| 3208 | return; |
| 3209 | } |
| 3210 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3211 | if (Srcs.hasConstOperand()) { |
| 3212 | if (!Srcs.immediateIsFlexEncodable() && |
| 3213 | Srcs.negatedImmediateIsFlexEncodable()) { |
Eric Holk | 40c69b4 | 2016-01-26 10:10:39 -0800 | [diff] [blame] | 3214 | assert(!isVectorType(DestTy)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3215 | Variable *Src0R = Srcs.src0R(this); |
| 3216 | Operand *Src1F = Srcs.negatedSrc1F(this); |
| 3217 | if (!Srcs.swappedOperands()) { |
| 3218 | _sub(T, Src0R, Src1F); |
| 3219 | } else { |
| 3220 | _rsb(T, Src0R, Src1F); |
| 3221 | } |
| 3222 | _mov(Dest, T); |
| 3223 | return; |
| 3224 | } |
| 3225 | } |
| 3226 | Variable *Src0R = Srcs.src0R(this); |
Eric Holk | 40c69b4 | 2016-01-26 10:10:39 -0800 | [diff] [blame] | 3227 | if (isVectorType(DestTy)) { |
| 3228 | Variable *Src1R = legalizeToReg(Src1); |
| 3229 | _vadd(T, Src0R, Src1R); |
| 3230 | } else { |
| 3231 | Operand *Src1RF = Srcs.src1RF(this); |
| 3232 | _add(T, Src0R, Src1RF); |
| 3233 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3234 | _mov(Dest, T); |
| 3235 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3236 | } |
| 3237 | case InstArithmetic::And: { |
| 3238 | if (Srcs.hasConstOperand()) { |
| 3239 | if (!Srcs.immediateIsFlexEncodable() && |
| 3240 | Srcs.invertedImmediateIsFlexEncodable()) { |
| 3241 | Variable *Src0R = Srcs.src0R(this); |
| 3242 | Operand *Src1F = Srcs.invertedSrc1F(this); |
| 3243 | _bic(T, Src0R, Src1F); |
| 3244 | _mov(Dest, T); |
| 3245 | return; |
| 3246 | } |
| 3247 | } |
Karl Schimpf | fd7975f | 2016-02-02 11:25:10 -0800 | [diff] [blame] | 3248 | assert(isIntegerType(DestTy)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3249 | Variable *Src0R = Srcs.src0R(this); |
Eric Holk | b58170c | 2016-01-27 11:18:29 -0800 | [diff] [blame] | 3250 | if (isVectorType(DestTy)) { |
| 3251 | Variable *Src1R = legalizeToReg(Src1); |
| 3252 | _vand(T, Src0R, Src1R); |
| 3253 | } else { |
| 3254 | Operand *Src1RF = Srcs.src1RF(this); |
| 3255 | _and(T, Src0R, Src1RF); |
| 3256 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3257 | _mov(Dest, T); |
| 3258 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3259 | } |
| 3260 | case InstArithmetic::Or: { |
| 3261 | Variable *Src0R = Srcs.src0R(this); |
Karl Schimpf | e295575 | 2016-02-02 12:57:30 -0800 | [diff] [blame] | 3262 | assert(isIntegerType(DestTy)); |
Eric Holk | cad0b75 | 2016-01-27 14:56:22 -0800 | [diff] [blame] | 3263 | if (isVectorType(DestTy)) { |
| 3264 | Variable *Src1R = legalizeToReg(Src1); |
| 3265 | _vorr(T, Src0R, Src1R); |
| 3266 | } else { |
| 3267 | Operand *Src1RF = Srcs.src1RF(this); |
| 3268 | _orr(T, Src0R, Src1RF); |
| 3269 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3270 | _mov(Dest, T); |
| 3271 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3272 | } |
| 3273 | case InstArithmetic::Xor: { |
| 3274 | Variable *Src0R = Srcs.src0R(this); |
Karl Schimpf | 625dfb3 | 2016-02-03 13:21:50 -0800 | [diff] [blame] | 3275 | assert(isIntegerType(DestTy)); |
Eric Holk | 76108e9 | 2016-01-28 13:37:50 -0800 | [diff] [blame] | 3276 | if (isVectorType(DestTy)) { |
| 3277 | Variable *Src1R = legalizeToReg(Src1); |
| 3278 | _veor(T, Src0R, Src1R); |
| 3279 | } else { |
| 3280 | Operand *Src1RF = Srcs.src1RF(this); |
| 3281 | _eor(T, Src0R, Src1RF); |
| 3282 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3283 | _mov(Dest, T); |
| 3284 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3285 | } |
| 3286 | case InstArithmetic::Sub: { |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3287 | if (const Inst *Src1Producer = Computations.getProducerOf(Src1)) { |
Eric Holk | e5727b8 | 2016-01-27 11:08:48 -0800 | [diff] [blame] | 3288 | assert(!isVectorType(DestTy)); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3289 | Variable *Src0R = legalizeToReg(Src0); |
| 3290 | Variable *Src1R = legalizeToReg(Src1Producer->getSrc(0)); |
| 3291 | Variable *Src2R = legalizeToReg(Src1Producer->getSrc(1)); |
| 3292 | _mls(T, Src1R, Src2R, Src0R); |
| 3293 | _mov(Dest, T); |
| 3294 | return; |
| 3295 | } |
| 3296 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3297 | if (Srcs.hasConstOperand()) { |
Eric Holk | e5727b8 | 2016-01-27 11:08:48 -0800 | [diff] [blame] | 3298 | assert(!isVectorType(DestTy)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3299 | if (Srcs.immediateIsFlexEncodable()) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 3300 | Variable *Src0R = Srcs.src0R(this); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3301 | Operand *Src1RF = Srcs.src1RF(this); |
| 3302 | if (Srcs.swappedOperands()) { |
| 3303 | _rsb(T, Src0R, Src1RF); |
| 3304 | } else { |
| 3305 | _sub(T, Src0R, Src1RF); |
| 3306 | } |
| 3307 | _mov(Dest, T); |
| 3308 | return; |
| 3309 | } |
| 3310 | if (!Srcs.swappedOperands() && Srcs.negatedImmediateIsFlexEncodable()) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 3311 | Variable *Src0R = Srcs.src0R(this); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3312 | Operand *Src1F = Srcs.negatedSrc1F(this); |
| 3313 | _add(T, Src0R, Src1F); |
| 3314 | _mov(Dest, T); |
| 3315 | return; |
| 3316 | } |
| 3317 | } |
| 3318 | Variable *Src0R = Srcs.unswappedSrc0R(this); |
| 3319 | Variable *Src1R = Srcs.unswappedSrc1R(this); |
Eric Holk | e5727b8 | 2016-01-27 11:08:48 -0800 | [diff] [blame] | 3320 | if (isVectorType(DestTy)) { |
| 3321 | _vsub(T, Src0R, Src1R); |
| 3322 | } else { |
| 3323 | _sub(T, Src0R, Src1R); |
| 3324 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3325 | _mov(Dest, T); |
| 3326 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3327 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3328 | case InstArithmetic::Mul: { |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 3329 | const bool OptM1 = getFlags().getOptLevel() == Opt_m1; |
John Porto | 98cc08c | 2015-11-24 12:30:01 -0800 | [diff] [blame] | 3330 | if (!OptM1 && Srcs.hasConstOperand()) { |
| 3331 | constexpr std::size_t MaxShifts = 4; |
| 3332 | std::array<StrengthReduction::AggregationElement, MaxShifts> Shifts; |
| 3333 | SizeT NumOperations; |
| 3334 | int32_t Const = Srcs.getConstantValue(); |
| 3335 | const bool Invert = Const < 0; |
| 3336 | const bool MultiplyByZero = Const == 0; |
| 3337 | Operand *_0 = |
| 3338 | legalize(Ctx->getConstantZero(DestTy), Legal_Reg | Legal_Flex); |
| 3339 | |
| 3340 | if (MultiplyByZero) { |
| 3341 | _mov(T, _0); |
| 3342 | _mov(Dest, T); |
| 3343 | return; |
| 3344 | } |
| 3345 | |
| 3346 | if (Invert) { |
| 3347 | Const = -Const; |
| 3348 | } |
| 3349 | |
| 3350 | if (StrengthReduction::tryToOptimize(Const, &NumOperations, &Shifts)) { |
| 3351 | assert(NumOperations >= 1); |
| 3352 | Variable *Src0R = Srcs.src0R(this); |
| 3353 | int32_t Start; |
| 3354 | int32_t End; |
| 3355 | if (NumOperations == 1 || Shifts[NumOperations - 1].shAmt() != 0) { |
| 3356 | // Multiplication by a power of 2 (NumOperations == 1); or |
| 3357 | // Multiplication by a even number not a power of 2. |
| 3358 | Start = 1; |
| 3359 | End = NumOperations; |
| 3360 | assert(Shifts[0].aggregateWithAdd()); |
| 3361 | _lsl(T, Src0R, shAmtImm(Shifts[0].shAmt())); |
| 3362 | } else { |
| 3363 | // Multiplication by an odd number. Put the free barrel shifter to a |
| 3364 | // good use. |
| 3365 | Start = 0; |
| 3366 | End = NumOperations - 2; |
| 3367 | const StrengthReduction::AggregationElement &Last = |
| 3368 | Shifts[NumOperations - 1]; |
| 3369 | const StrengthReduction::AggregationElement &SecondToLast = |
| 3370 | Shifts[NumOperations - 2]; |
| 3371 | if (!Last.aggregateWithAdd()) { |
| 3372 | assert(SecondToLast.aggregateWithAdd()); |
| 3373 | _rsb(T, Src0R, SecondToLast.createShiftedOperand(Func, Src0R)); |
| 3374 | } else if (!SecondToLast.aggregateWithAdd()) { |
| 3375 | assert(Last.aggregateWithAdd()); |
| 3376 | _sub(T, Src0R, SecondToLast.createShiftedOperand(Func, Src0R)); |
| 3377 | } else { |
| 3378 | _add(T, Src0R, SecondToLast.createShiftedOperand(Func, Src0R)); |
| 3379 | } |
| 3380 | } |
| 3381 | |
| 3382 | // Odd numbers : S E I I |
| 3383 | // +---+---+---+---+---+---+ ... +---+---+---+---+ |
| 3384 | // Shifts = | | | | | | | ... | | | | | |
| 3385 | // +---+---+---+---+---+---+ ... +---+---+---+---+ |
| 3386 | // Even numbers: I S E |
| 3387 | // |
| 3388 | // S: Start; E: End; I: Init |
| 3389 | for (int32_t I = Start; I < End; ++I) { |
| 3390 | const StrengthReduction::AggregationElement &Current = Shifts[I]; |
| 3391 | Operand *SrcF = Current.createShiftedOperand(Func, Src0R); |
| 3392 | if (Current.aggregateWithAdd()) { |
| 3393 | _add(T, T, SrcF); |
| 3394 | } else { |
| 3395 | _sub(T, T, SrcF); |
| 3396 | } |
| 3397 | } |
| 3398 | |
| 3399 | if (Invert) { |
| 3400 | // T = 0 - T. |
| 3401 | _rsb(T, T, _0); |
| 3402 | } |
| 3403 | |
| 3404 | _mov(Dest, T); |
| 3405 | return; |
| 3406 | } |
| 3407 | } |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3408 | Variable *Src0R = Srcs.unswappedSrc0R(this); |
| 3409 | Variable *Src1R = Srcs.unswappedSrc1R(this); |
Eric Holk | 029bed9 | 2016-01-28 13:38:43 -0800 | [diff] [blame] | 3410 | if (isVectorType(DestTy)) { |
| 3411 | _vmul(T, Src0R, Src1R); |
| 3412 | } else { |
| 3413 | _mul(T, Src0R, Src1R); |
| 3414 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3415 | _mov(Dest, T); |
| 3416 | return; |
| 3417 | } |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3418 | case InstArithmetic::Shl: { |
| 3419 | Variable *Src0R = Srcs.unswappedSrc0R(this); |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 3420 | if (!isVectorType(T->getType())) { |
| 3421 | Operand *Src1R = Srcs.unswappedSrc1RShAmtImm(this); |
| 3422 | _lsl(T, Src0R, Src1R); |
| 3423 | } else { |
| 3424 | auto *Src1R = Srcs.unswappedSrc1R(this); |
| 3425 | _vshl(T, Src0R, Src1R)->setSignType(InstARM32::FS_Unsigned); |
| 3426 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3427 | _mov(Dest, T); |
| 3428 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3429 | } |
| 3430 | case InstArithmetic::Lshr: { |
| 3431 | Variable *Src0R = Srcs.unswappedSrc0R(this); |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 3432 | if (!isVectorType(T->getType())) { |
| 3433 | Operand *Src1R = Srcs.unswappedSrc1RShAmtImm(this); |
| 3434 | if (DestTy != IceType_i32) { |
| 3435 | _uxt(Src0R, Src0R); |
| 3436 | } |
| 3437 | _lsr(T, Src0R, Src1R); |
| 3438 | } else { |
| 3439 | auto *Src1R = Srcs.unswappedSrc1R(this); |
| 3440 | auto *Src1RNeg = makeReg(Src1R->getType()); |
| 3441 | _vneg(Src1RNeg, Src1R); |
| 3442 | _vshl(T, Src0R, Src1RNeg)->setSignType(InstARM32::FS_Unsigned); |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 3443 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3444 | _mov(Dest, T); |
| 3445 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3446 | } |
| 3447 | case InstArithmetic::Ashr: { |
| 3448 | Variable *Src0R = Srcs.unswappedSrc0R(this); |
John Porto | 15e77d4 | 2016-04-13 12:57:14 -0700 | [diff] [blame] | 3449 | if (!isVectorType(T->getType())) { |
| 3450 | if (DestTy != IceType_i32) { |
| 3451 | _sxt(Src0R, Src0R); |
| 3452 | } |
| 3453 | _asr(T, Src0R, Srcs.unswappedSrc1RShAmtImm(this)); |
| 3454 | } else { |
| 3455 | auto *Src1R = Srcs.unswappedSrc1R(this); |
| 3456 | auto *Src1RNeg = makeReg(Src1R->getType()); |
| 3457 | _vneg(Src1RNeg, Src1R); |
| 3458 | _vshl(T, Src0R, Src1RNeg)->setSignType(InstARM32::FS_Signed); |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 3459 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3460 | _mov(Dest, T); |
| 3461 | return; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3462 | } |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3463 | case InstArithmetic::Udiv: |
| 3464 | case InstArithmetic::Sdiv: |
| 3465 | case InstArithmetic::Urem: |
| 3466 | case InstArithmetic::Srem: |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3467 | llvm::report_fatal_error( |
| 3468 | "Integer div/rem should have been handled earlier."); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3469 | return; |
| 3470 | case InstArithmetic::Fadd: |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3471 | case InstArithmetic::Fsub: |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3472 | case InstArithmetic::Fmul: |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3473 | case InstArithmetic::Fdiv: |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3474 | case InstArithmetic::Frem: |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3475 | llvm::report_fatal_error( |
| 3476 | "Floating point arith should have been handled earlier."); |
Jan Voung | 70fa525 | 2015-07-06 14:01:25 -0700 | [diff] [blame] | 3477 | return; |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 3478 | } |
| 3479 | } |
| 3480 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 3481 | void TargetARM32::lowerAssign(const InstAssign *Instr) { |
| 3482 | Variable *Dest = Instr->getDest(); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 3483 | |
| 3484 | if (Dest->isRematerializable()) { |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 3485 | Context.insert<InstFakeDef>(Dest); |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 3486 | return; |
| 3487 | } |
| 3488 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 3489 | Operand *Src0 = Instr->getSrc(0); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 3490 | assert(Dest->getType() == Src0->getType()); |
| 3491 | if (Dest->getType() == IceType_i64) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 3492 | Src0 = legalizeUndef(Src0); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 3493 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3494 | Variable *T_Lo = makeReg(IceType_i32); |
| 3495 | auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 3496 | Operand *Src0Lo = legalize(loOperand(Src0), Legal_Reg | Legal_Flex); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 3497 | _mov(T_Lo, Src0Lo); |
| 3498 | _mov(DestLo, T_Lo); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3499 | |
| 3500 | Variable *T_Hi = makeReg(IceType_i32); |
| 3501 | auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 3502 | Operand *Src0Hi = legalize(hiOperand(Src0), Legal_Reg | Legal_Flex); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 3503 | _mov(T_Hi, Src0Hi); |
| 3504 | _mov(DestHi, T_Hi); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3505 | |
| 3506 | return; |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 3507 | } |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 3508 | |
| 3509 | Operand *NewSrc; |
| 3510 | if (Dest->hasReg()) { |
| 3511 | // If Dest already has a physical register, then legalize the Src operand |
| 3512 | // into a Variable with the same register assignment. This especially |
| 3513 | // helps allow the use of Flex operands. |
| 3514 | NewSrc = legalize(Src0, Legal_Reg | Legal_Flex, Dest->getRegNum()); |
| 3515 | } else { |
| 3516 | // Dest could be a stack operand. Since we could potentially need to do a |
| 3517 | // Store (and store can only have Register operands), legalize this to a |
| 3518 | // register. |
| 3519 | NewSrc = legalize(Src0, Legal_Reg); |
| 3520 | } |
| 3521 | |
| 3522 | if (isVectorType(Dest->getType()) || isScalarFloatingType(Dest->getType())) { |
| 3523 | NewSrc = legalize(NewSrc, Legal_Reg | Legal_Mem); |
| 3524 | } |
| 3525 | _mov(Dest, NewSrc); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 3526 | } |
| 3527 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3528 | TargetARM32::ShortCircuitCondAndLabel TargetARM32::lowerInt1ForBranch( |
| 3529 | Operand *Boolean, const LowerInt1BranchTarget &TargetTrue, |
| 3530 | const LowerInt1BranchTarget &TargetFalse, uint32_t ShortCircuitable) { |
| 3531 | InstARM32Label *NewShortCircuitLabel = nullptr; |
| 3532 | Operand *_1 = legalize(Ctx->getConstantInt1(1), Legal_Reg | Legal_Flex); |
| 3533 | |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3534 | const Inst *Producer = Computations.getProducerOf(Boolean); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3535 | |
| 3536 | if (Producer == nullptr) { |
| 3537 | // No producer, no problem: just do emit code to perform (Boolean & 1) and |
| 3538 | // set the flags register. The branch should be taken if the resulting flags |
| 3539 | // indicate a non-zero result. |
| 3540 | _tst(legalizeToReg(Boolean), _1); |
| 3541 | return ShortCircuitCondAndLabel(CondWhenTrue(CondARM32::NE)); |
| 3542 | } |
| 3543 | |
| 3544 | switch (Producer->getKind()) { |
| 3545 | default: |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3546 | llvm::report_fatal_error("Unexpected producer."); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3547 | case Inst::Icmp: { |
| 3548 | return ShortCircuitCondAndLabel( |
| 3549 | lowerIcmpCond(llvm::cast<InstIcmp>(Producer))); |
| 3550 | } break; |
| 3551 | case Inst::Fcmp: { |
| 3552 | return ShortCircuitCondAndLabel( |
| 3553 | lowerFcmpCond(llvm::cast<InstFcmp>(Producer))); |
| 3554 | } break; |
| 3555 | case Inst::Cast: { |
| 3556 | const auto *CastProducer = llvm::cast<InstCast>(Producer); |
| 3557 | assert(CastProducer->getCastKind() == InstCast::Trunc); |
| 3558 | Operand *Src = CastProducer->getSrc(0); |
| 3559 | if (Src->getType() == IceType_i64) |
| 3560 | Src = loOperand(Src); |
| 3561 | _tst(legalizeToReg(Src), _1); |
| 3562 | return ShortCircuitCondAndLabel(CondWhenTrue(CondARM32::NE)); |
| 3563 | } break; |
| 3564 | case Inst::Arithmetic: { |
| 3565 | const auto *ArithProducer = llvm::cast<InstArithmetic>(Producer); |
| 3566 | switch (ArithProducer->getOp()) { |
| 3567 | default: |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3568 | llvm::report_fatal_error("Unhandled Arithmetic Producer."); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3569 | case InstArithmetic::And: { |
| 3570 | if (!(ShortCircuitable & SC_And)) { |
| 3571 | NewShortCircuitLabel = InstARM32Label::create(Func, this); |
| 3572 | } |
| 3573 | |
| 3574 | LowerInt1BranchTarget NewTarget = |
| 3575 | TargetFalse.createForLabelOrDuplicate(NewShortCircuitLabel); |
| 3576 | |
| 3577 | ShortCircuitCondAndLabel CondAndLabel = lowerInt1ForBranch( |
| 3578 | Producer->getSrc(0), TargetTrue, NewTarget, SC_And); |
| 3579 | const CondWhenTrue &Cond = CondAndLabel.Cond; |
| 3580 | |
| 3581 | _br_short_circuit(NewTarget, Cond.invert()); |
| 3582 | |
| 3583 | InstARM32Label *const ShortCircuitLabel = CondAndLabel.ShortCircuitTarget; |
| 3584 | if (ShortCircuitLabel != nullptr) |
| 3585 | Context.insert(ShortCircuitLabel); |
| 3586 | |
| 3587 | return ShortCircuitCondAndLabel( |
| 3588 | lowerInt1ForBranch(Producer->getSrc(1), TargetTrue, NewTarget, SC_All) |
| 3589 | .assertNoLabelAndReturnCond(), |
| 3590 | NewShortCircuitLabel); |
| 3591 | } break; |
| 3592 | case InstArithmetic::Or: { |
| 3593 | if (!(ShortCircuitable & SC_Or)) { |
| 3594 | NewShortCircuitLabel = InstARM32Label::create(Func, this); |
| 3595 | } |
| 3596 | |
| 3597 | LowerInt1BranchTarget NewTarget = |
| 3598 | TargetTrue.createForLabelOrDuplicate(NewShortCircuitLabel); |
| 3599 | |
| 3600 | ShortCircuitCondAndLabel CondAndLabel = lowerInt1ForBranch( |
| 3601 | Producer->getSrc(0), NewTarget, TargetFalse, SC_Or); |
| 3602 | const CondWhenTrue &Cond = CondAndLabel.Cond; |
| 3603 | |
| 3604 | _br_short_circuit(NewTarget, Cond); |
| 3605 | |
| 3606 | InstARM32Label *const ShortCircuitLabel = CondAndLabel.ShortCircuitTarget; |
| 3607 | if (ShortCircuitLabel != nullptr) |
| 3608 | Context.insert(ShortCircuitLabel); |
| 3609 | |
| 3610 | return ShortCircuitCondAndLabel(lowerInt1ForBranch(Producer->getSrc(1), |
| 3611 | NewTarget, TargetFalse, |
| 3612 | SC_All) |
| 3613 | .assertNoLabelAndReturnCond(), |
| 3614 | NewShortCircuitLabel); |
| 3615 | } break; |
| 3616 | } |
| 3617 | } |
| 3618 | } |
| 3619 | } |
| 3620 | |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3621 | void TargetARM32::lowerBr(const InstBr *Instr) { |
| 3622 | if (Instr->isUnconditional()) { |
| 3623 | _br(Instr->getTargetUnconditional()); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3624 | return; |
| 3625 | } |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3626 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3627 | CfgNode *TargetTrue = Instr->getTargetTrue(); |
| 3628 | CfgNode *TargetFalse = Instr->getTargetFalse(); |
| 3629 | ShortCircuitCondAndLabel CondAndLabel = lowerInt1ForBranch( |
| 3630 | Instr->getCondition(), LowerInt1BranchTarget(TargetTrue), |
| 3631 | LowerInt1BranchTarget(TargetFalse), SC_All); |
| 3632 | assert(CondAndLabel.ShortCircuitTarget == nullptr); |
| 3633 | |
| 3634 | const CondWhenTrue &Cond = CondAndLabel.Cond; |
| 3635 | if (Cond.WhenTrue1 != CondARM32::kNone) { |
| 3636 | assert(Cond.WhenTrue0 != CondARM32::AL); |
| 3637 | _br(TargetTrue, Cond.WhenTrue1); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3638 | } |
| 3639 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3640 | switch (Cond.WhenTrue0) { |
| 3641 | default: |
| 3642 | _br(TargetTrue, TargetFalse, Cond.WhenTrue0); |
| 3643 | break; |
| 3644 | case CondARM32::kNone: |
| 3645 | _br(TargetFalse); |
| 3646 | break; |
| 3647 | case CondARM32::AL: |
| 3648 | _br(TargetTrue); |
| 3649 | break; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3650 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 3651 | } |
| 3652 | |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3653 | void TargetARM32::lowerCall(const InstCall *Instr) { |
John Porto | b82d79a | 2016-03-01 06:11:05 -0800 | [diff] [blame] | 3654 | Operand *CallTarget = Instr->getCallTarget(); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3655 | if (Instr->isTargetHelperCall()) { |
| 3656 | auto TargetHelperPreamble = ARM32HelpersPreamble.find(CallTarget); |
| 3657 | if (TargetHelperPreamble != ARM32HelpersPreamble.end()) { |
| 3658 | (this->*TargetHelperPreamble->second)(Instr); |
| 3659 | } |
| 3660 | } |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 3661 | MaybeLeafFunc = false; |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3662 | NeedsStackAlignment = true; |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 3663 | |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3664 | // Assign arguments to registers and stack. Also reserve stack. |
| 3665 | TargetARM32::CallingConv CC; |
| 3666 | // Pair of Arg Operand -> GPR number assignments. |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 3667 | llvm::SmallVector<std::pair<Operand *, RegNumT>, NumGPRArgs> GPRArgs; |
| 3668 | llvm::SmallVector<std::pair<Operand *, RegNumT>, NumFP32Args> FPArgs; |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3669 | // Pair of Arg Operand -> stack offset. |
| 3670 | llvm::SmallVector<std::pair<Operand *, int32_t>, 8> StackArgs; |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 3671 | size_t ParameterAreaSizeBytes = 0; |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3672 | |
| 3673 | // Classify each argument operand according to the location where the |
| 3674 | // argument is passed. |
| 3675 | for (SizeT i = 0, NumArgs = Instr->getNumArgs(); i < NumArgs; ++i) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 3676 | Operand *Arg = legalizeUndef(Instr->getArg(i)); |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 3677 | const Type Ty = Arg->getType(); |
| 3678 | bool InReg = false; |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 3679 | RegNumT Reg; |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 3680 | if (isScalarIntegerType(Ty)) { |
| 3681 | InReg = CC.argInGPR(Ty, &Reg); |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3682 | } else { |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 3683 | InReg = CC.argInVFP(Ty, &Reg); |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3684 | } |
| 3685 | |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 3686 | if (!InReg) { |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3687 | ParameterAreaSizeBytes = |
| 3688 | applyStackAlignmentTy(ParameterAreaSizeBytes, Ty); |
| 3689 | StackArgs.push_back(std::make_pair(Arg, ParameterAreaSizeBytes)); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3690 | ParameterAreaSizeBytes += typeWidthInBytesOnStack(Ty); |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 3691 | continue; |
| 3692 | } |
| 3693 | |
| 3694 | if (Ty == IceType_i64) { |
| 3695 | Operand *Lo = loOperand(Arg); |
| 3696 | Operand *Hi = hiOperand(Arg); |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 3697 | GPRArgs.push_back(std::make_pair( |
| 3698 | Lo, RegNumT::fixme(RegARM32::getI64PairFirstGPRNum(Reg)))); |
| 3699 | GPRArgs.push_back(std::make_pair( |
| 3700 | Hi, RegNumT::fixme(RegARM32::getI64PairSecondGPRNum(Reg)))); |
John Porto | 2187c84 | 2015-12-16 07:48:25 -0800 | [diff] [blame] | 3701 | } else if (isScalarIntegerType(Ty)) { |
| 3702 | GPRArgs.push_back(std::make_pair(Arg, Reg)); |
| 3703 | } else { |
| 3704 | FPArgs.push_back(std::make_pair(Arg, Reg)); |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3705 | } |
| 3706 | } |
| 3707 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 3708 | // Adjust the parameter area so that the stack is aligned. It is assumed that |
| 3709 | // the stack is already aligned at the start of the calling sequence. |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3710 | ParameterAreaSizeBytes = applyStackAlignment(ParameterAreaSizeBytes); |
| 3711 | |
John Porto | f419854 | 2015-11-20 14:17:23 -0800 | [diff] [blame] | 3712 | if (ParameterAreaSizeBytes > MaxOutArgsSizeBytes) { |
| 3713 | llvm::report_fatal_error("MaxOutArgsSizeBytes is not really a max."); |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3714 | } |
| 3715 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 3716 | // Copy arguments that are passed on the stack to the appropriate stack |
| 3717 | // locations. |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 3718 | Variable *SP = getPhysicalRegister(RegARM32::Reg_sp); |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3719 | for (auto &StackArg : StackArgs) { |
| 3720 | ConstantInteger32 *Loc = |
| 3721 | llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(StackArg.second)); |
| 3722 | Type Ty = StackArg.first->getType(); |
| 3723 | OperandARM32Mem *Addr; |
| 3724 | constexpr bool SignExt = false; |
| 3725 | if (OperandARM32Mem::canHoldOffset(Ty, SignExt, StackArg.second)) { |
| 3726 | Addr = OperandARM32Mem::create(Func, Ty, SP, Loc); |
| 3727 | } else { |
| 3728 | Variable *NewBase = Func->makeVariable(SP->getType()); |
| 3729 | lowerArithmetic( |
| 3730 | InstArithmetic::create(Func, InstArithmetic::Add, NewBase, SP, Loc)); |
| 3731 | Addr = formMemoryOperand(NewBase, Ty); |
| 3732 | } |
| 3733 | lowerStore(InstStore::create(Func, StackArg.first, Addr)); |
| 3734 | } |
| 3735 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 3736 | // Generate the call instruction. Assign its result to a temporary with high |
| 3737 | // register allocation weight. |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3738 | Variable *Dest = Instr->getDest(); |
| 3739 | // ReturnReg doubles as ReturnRegLo as necessary. |
| 3740 | Variable *ReturnReg = nullptr; |
| 3741 | Variable *ReturnRegHi = nullptr; |
| 3742 | if (Dest) { |
| 3743 | switch (Dest->getType()) { |
| 3744 | case IceType_NUM: |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3745 | llvm::report_fatal_error("Invalid Call dest type"); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3746 | break; |
| 3747 | case IceType_void: |
| 3748 | break; |
| 3749 | case IceType_i1: |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 3750 | assert(Computations.getProducerOf(Dest) == nullptr); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3751 | // Fall-through intended. |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3752 | case IceType_i8: |
| 3753 | case IceType_i16: |
| 3754 | case IceType_i32: |
| 3755 | ReturnReg = makeReg(Dest->getType(), RegARM32::Reg_r0); |
| 3756 | break; |
| 3757 | case IceType_i64: |
| 3758 | ReturnReg = makeReg(IceType_i32, RegARM32::Reg_r0); |
| 3759 | ReturnRegHi = makeReg(IceType_i32, RegARM32::Reg_r1); |
| 3760 | break; |
| 3761 | case IceType_f32: |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3762 | ReturnReg = makeReg(Dest->getType(), RegARM32::Reg_s0); |
| 3763 | break; |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3764 | case IceType_f64: |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3765 | ReturnReg = makeReg(Dest->getType(), RegARM32::Reg_d0); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3766 | break; |
| 3767 | case IceType_v4i1: |
| 3768 | case IceType_v8i1: |
| 3769 | case IceType_v16i1: |
| 3770 | case IceType_v16i8: |
| 3771 | case IceType_v8i16: |
| 3772 | case IceType_v4i32: |
| 3773 | case IceType_v4f32: |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 3774 | ReturnReg = makeReg(Dest->getType(), RegARM32::Reg_q0); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3775 | break; |
| 3776 | } |
| 3777 | } |
Jan Voung | b0a8c24 | 2015-06-18 15:00:14 -0700 | [diff] [blame] | 3778 | |
John Porto | b82d79a | 2016-03-01 06:11:05 -0800 | [diff] [blame] | 3779 | // Allow ConstantRelocatable to be left alone as a direct call, but force |
| 3780 | // other constants like ConstantInteger32 to be in a register and make it an |
| 3781 | // indirect call. |
| 3782 | if (!llvm::isa<ConstantRelocatable>(CallTarget)) { |
| 3783 | CallTarget = legalize(CallTarget, Legal_Reg); |
| 3784 | } |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3785 | |
| 3786 | // Copy arguments to be passed in registers to the appropriate registers. |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 3787 | CfgVector<Variable *> RegArgs; |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3788 | for (auto &FPArg : FPArgs) { |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 3789 | RegArgs.emplace_back(legalizeToReg(FPArg.first, FPArg.second)); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3790 | } |
| 3791 | for (auto &GPRArg : GPRArgs) { |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 3792 | RegArgs.emplace_back(legalizeToReg(GPRArg.first, GPRArg.second)); |
| 3793 | } |
| 3794 | |
| 3795 | // Generate a FakeUse of register arguments so that they do not get dead code |
| 3796 | // eliminated as a result of the FakeKill of scratch registers after the call. |
| 3797 | // These fake-uses need to be placed here to avoid argument registers from |
| 3798 | // being used during the legalizeToReg() calls above. |
| 3799 | for (auto *RegArg : RegArgs) { |
| 3800 | Context.insert<InstFakeUse>(RegArg); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3801 | } |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 3802 | |
| 3803 | InstARM32Call *NewCall = |
| 3804 | Sandboxer(this, InstBundleLock::Opt_AlignToEnd).bl(ReturnReg, CallTarget); |
| 3805 | |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3806 | if (ReturnRegHi) |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 3807 | Context.insert<InstFakeDef>(ReturnRegHi); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3808 | |
| 3809 | // Insert a register-kill pseudo instruction. |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 3810 | Context.insert<InstFakeKill>(NewCall); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3811 | |
| 3812 | // Generate a FakeUse to keep the call live if necessary. |
| 3813 | if (Instr->hasSideEffects() && ReturnReg) { |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 3814 | Context.insert<InstFakeUse>(ReturnReg); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3815 | } |
| 3816 | |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3817 | if (Dest != nullptr) { |
| 3818 | // Assign the result of the call to Dest. |
| 3819 | if (ReturnReg != nullptr) { |
| 3820 | if (ReturnRegHi) { |
| 3821 | auto *Dest64On32 = llvm::cast<Variable64On32>(Dest); |
| 3822 | Variable *DestLo = Dest64On32->getLo(); |
| 3823 | Variable *DestHi = Dest64On32->getHi(); |
| 3824 | _mov(DestLo, ReturnReg); |
| 3825 | _mov(DestHi, ReturnRegHi); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3826 | } else { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3827 | if (isFloatingType(Dest->getType()) || isVectorType(Dest->getType())) { |
| 3828 | _mov(Dest, ReturnReg); |
| 3829 | } else { |
| 3830 | assert(isIntegerType(Dest->getType()) && |
| 3831 | typeWidthInBytes(Dest->getType()) <= 4); |
| 3832 | _mov(Dest, ReturnReg); |
| 3833 | } |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 3834 | } |
| 3835 | } |
| 3836 | } |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3837 | |
| 3838 | if (Instr->isTargetHelperCall()) { |
John Porto | b82d79a | 2016-03-01 06:11:05 -0800 | [diff] [blame] | 3839 | auto TargetHelpersPostamble = ARM32HelpersPostamble.find(CallTarget); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 3840 | if (TargetHelpersPostamble != ARM32HelpersPostamble.end()) { |
| 3841 | (this->*TargetHelpersPostamble->second)(Instr); |
| 3842 | } |
| 3843 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 3844 | } |
| 3845 | |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3846 | namespace { |
John Porto | e0b829f | 2015-09-28 09:50:48 -0700 | [diff] [blame] | 3847 | void configureBitcastTemporary(Variable64On32 *Var) { |
| 3848 | Var->setMustNotHaveReg(); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 3849 | Var->getHi()->setMustHaveReg(); |
| 3850 | Var->getLo()->setMustHaveReg(); |
| 3851 | } |
| 3852 | } // end of anonymous namespace |
| 3853 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 3854 | void TargetARM32::lowerCast(const InstCast *Instr) { |
| 3855 | InstCast::OpKind CastKind = Instr->getCastKind(); |
| 3856 | Variable *Dest = Instr->getDest(); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3857 | const Type DestTy = Dest->getType(); |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 3858 | Operand *Src0 = legalizeUndef(Instr->getSrc(0)); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 3859 | switch (CastKind) { |
| 3860 | default: |
| 3861 | Func->setError("Cast type not supported"); |
| 3862 | return; |
| 3863 | case InstCast::Sext: { |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3864 | if (isVectorType(DestTy)) { |
| 3865 | Variable *T0 = makeReg(DestTy); |
| 3866 | Variable *T1 = makeReg(DestTy); |
| 3867 | ConstantInteger32 *ShAmt = nullptr; |
| 3868 | switch (DestTy) { |
| 3869 | default: |
| 3870 | llvm::report_fatal_error("Unexpected type in vector sext."); |
| 3871 | case IceType_v16i8: |
| 3872 | ShAmt = llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(7)); |
| 3873 | break; |
| 3874 | case IceType_v8i16: |
| 3875 | ShAmt = llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(15)); |
| 3876 | break; |
| 3877 | case IceType_v4i32: |
| 3878 | ShAmt = llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(31)); |
| 3879 | break; |
| 3880 | } |
| 3881 | auto *Src0R = legalizeToReg(Src0); |
| 3882 | _vshl(T0, Src0R, ShAmt); |
| 3883 | _vshr(T1, T0, ShAmt)->setSignType(InstARM32::FS_Signed); |
| 3884 | _mov(Dest, T1); |
| 3885 | } else if (DestTy == IceType_i64) { |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3886 | // t1=sxtb src; t2= mov t1 asr #31; dst.lo=t1; dst.hi=t2 |
| 3887 | Constant *ShiftAmt = Ctx->getConstantInt32(31); |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 3888 | auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 3889 | auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3890 | Variable *T_Lo = makeReg(DestLo->getType()); |
| 3891 | if (Src0->getType() == IceType_i32) { |
| 3892 | Operand *Src0RF = legalize(Src0, Legal_Reg | Legal_Flex); |
| 3893 | _mov(T_Lo, Src0RF); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3894 | } else if (Src0->getType() != IceType_i1) { |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 3895 | Variable *Src0R = legalizeToReg(Src0); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3896 | _sxt(T_Lo, Src0R); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3897 | } else { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3898 | Operand *_0 = Ctx->getConstantZero(IceType_i32); |
| 3899 | Operand *_m1 = Ctx->getConstantInt32(-1); |
| 3900 | lowerInt1ForSelect(T_Lo, Src0, _m1, _0); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3901 | } |
| 3902 | _mov(DestLo, T_Lo); |
| 3903 | Variable *T_Hi = makeReg(DestHi->getType()); |
| 3904 | if (Src0->getType() != IceType_i1) { |
| 3905 | _mov(T_Hi, OperandARM32FlexReg::create(Func, IceType_i32, T_Lo, |
| 3906 | OperandARM32::ASR, ShiftAmt)); |
| 3907 | } else { |
| 3908 | // For i1, the asr instruction is already done above. |
| 3909 | _mov(T_Hi, T_Lo); |
| 3910 | } |
| 3911 | _mov(DestHi, T_Hi); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3912 | } else if (Src0->getType() != IceType_i1) { |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3913 | // t1 = sxt src; dst = t1 |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 3914 | Variable *Src0R = legalizeToReg(Src0); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3915 | Variable *T = makeReg(DestTy); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3916 | _sxt(T, Src0R); |
| 3917 | _mov(Dest, T); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3918 | } else { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3919 | Constant *_0 = Ctx->getConstantZero(IceType_i32); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3920 | Operand *_m1 = Ctx->getConstantInt(DestTy, -1); |
| 3921 | Variable *T = makeReg(DestTy); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3922 | lowerInt1ForSelect(T, Src0, _m1, _0); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3923 | _mov(Dest, T); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3924 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 3925 | break; |
| 3926 | } |
| 3927 | case InstCast::Zext: { |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3928 | if (isVectorType(DestTy)) { |
| 3929 | auto *Mask = makeReg(DestTy); |
| 3930 | auto *_1 = Ctx->getConstantInt32(1); |
| 3931 | auto *T = makeReg(DestTy); |
| 3932 | auto *Src0R = legalizeToReg(Src0); |
| 3933 | _mov(Mask, _1); |
| 3934 | _vand(T, Src0R, Mask); |
| 3935 | _mov(Dest, T); |
| 3936 | } else if (DestTy == IceType_i64) { |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3937 | // t1=uxtb src; dst.lo=t1; dst.hi=0 |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3938 | Operand *_0 = |
| 3939 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 3940 | auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 3941 | auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3942 | Variable *T_Lo = makeReg(DestLo->getType()); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3943 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3944 | switch (Src0->getType()) { |
| 3945 | default: { |
| 3946 | assert(Src0->getType() != IceType_i64); |
| 3947 | _uxt(T_Lo, legalizeToReg(Src0)); |
| 3948 | } break; |
| 3949 | case IceType_i32: { |
| 3950 | _mov(T_Lo, legalize(Src0, Legal_Reg | Legal_Flex)); |
| 3951 | } break; |
| 3952 | case IceType_i1: { |
| 3953 | SafeBoolChain Safe = lowerInt1(T_Lo, Src0); |
| 3954 | if (Safe == SBC_No) { |
| 3955 | Operand *_1 = |
| 3956 | legalize(Ctx->getConstantInt1(1), Legal_Reg | Legal_Flex); |
| 3957 | _and(T_Lo, T_Lo, _1); |
| 3958 | } |
| 3959 | } break; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3960 | } |
| 3961 | |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3962 | _mov(DestLo, T_Lo); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3963 | |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3964 | Variable *T_Hi = makeReg(DestLo->getType()); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3965 | _mov(T_Hi, _0); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3966 | _mov(DestHi, T_Hi); |
| 3967 | } else if (Src0->getType() == IceType_i1) { |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3968 | Variable *T = makeReg(DestTy); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3969 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 3970 | SafeBoolChain Safe = lowerInt1(T, Src0); |
| 3971 | if (Safe == SBC_No) { |
| 3972 | Operand *_1 = legalize(Ctx->getConstantInt1(1), Legal_Reg | Legal_Flex); |
| 3973 | _and(T, T, _1); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 3974 | } |
| 3975 | |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3976 | _mov(Dest, T); |
| 3977 | } else { |
| 3978 | // t1 = uxt src; dst = t1 |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 3979 | Variable *Src0R = legalizeToReg(Src0); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3980 | Variable *T = makeReg(DestTy); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3981 | _uxt(T, Src0R); |
| 3982 | _mov(Dest, T); |
| 3983 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 3984 | break; |
| 3985 | } |
| 3986 | case InstCast::Trunc: { |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3987 | if (isVectorType(DestTy)) { |
| 3988 | auto *T = makeReg(DestTy); |
| 3989 | auto *Src0R = legalizeToReg(Src0); |
| 3990 | _mov(T, Src0R); |
| 3991 | _mov(Dest, T); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3992 | } else { |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3993 | if (Src0->getType() == IceType_i64) |
| 3994 | Src0 = loOperand(Src0); |
| 3995 | Operand *Src0RF = legalize(Src0, Legal_Reg | Legal_Flex); |
| 3996 | // t1 = trunc Src0RF; Dest = t1 |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3997 | Variable *T = makeReg(DestTy); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 3998 | _mov(T, Src0RF); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 3999 | if (DestTy == IceType_i1) |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 4000 | _and(T, T, Ctx->getConstantInt1(1)); |
| 4001 | _mov(Dest, T); |
| 4002 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4003 | break; |
| 4004 | } |
| 4005 | case InstCast::Fptrunc: |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4006 | case InstCast::Fpext: { |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4007 | // fptrunc: dest.f32 = fptrunc src0.fp64 |
| 4008 | // fpext: dest.f64 = fptrunc src0.fp32 |
| 4009 | const bool IsTrunc = CastKind == InstCast::Fptrunc; |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4010 | assert(!isVectorType(DestTy)); |
| 4011 | assert(DestTy == (IsTrunc ? IceType_f32 : IceType_f64)); |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4012 | assert(Src0->getType() == (IsTrunc ? IceType_f64 : IceType_f32)); |
| 4013 | Variable *Src0R = legalizeToReg(Src0); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4014 | Variable *T = makeReg(DestTy); |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4015 | _vcvt(T, Src0R, IsTrunc ? InstARM32Vcvt::D2s : InstARM32Vcvt::S2d); |
| 4016 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4017 | break; |
| 4018 | } |
| 4019 | case InstCast::Fptosi: |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4020 | case InstCast::Fptoui: { |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4021 | const bool DestIsSigned = CastKind == InstCast::Fptosi; |
| 4022 | Variable *Src0R = legalizeToReg(Src0); |
| 4023 | |
| 4024 | if (isVectorType(DestTy)) { |
| 4025 | assert(typeElementType(Src0->getType()) == IceType_f32); |
| 4026 | auto *T = makeReg(DestTy); |
| 4027 | _vcvt(T, Src0R, |
| 4028 | DestIsSigned ? InstARM32Vcvt::Vs2si : InstARM32Vcvt::Vs2ui); |
| 4029 | _mov(Dest, T); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4030 | break; |
| 4031 | } |
| 4032 | |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4033 | const bool Src0IsF32 = isFloat32Asserting32Or64(Src0->getType()); |
| 4034 | if (llvm::isa<Variable64On32>(Dest)) { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 4035 | llvm::report_fatal_error("fp-to-i64 should have been pre-lowered."); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4036 | } |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4037 | // fptosi: |
| 4038 | // t1.fp = vcvt src0.fp |
| 4039 | // t2.i32 = vmov t1.fp |
| 4040 | // dest.int = conv t2.i32 @ Truncates the result if needed. |
| 4041 | // fptoui: |
| 4042 | // t1.fp = vcvt src0.fp |
| 4043 | // t2.u32 = vmov t1.fp |
| 4044 | // dest.uint = conv t2.u32 @ Truncates the result if needed. |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4045 | Variable *T_fp = makeReg(IceType_f32); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4046 | const InstARM32Vcvt::VcvtVariant Conversion = |
| 4047 | Src0IsF32 ? (DestIsSigned ? InstARM32Vcvt::S2si : InstARM32Vcvt::S2ui) |
| 4048 | : (DestIsSigned ? InstARM32Vcvt::D2si : InstARM32Vcvt::D2ui); |
| 4049 | _vcvt(T_fp, Src0R, Conversion); |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4050 | Variable *T = makeReg(IceType_i32); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4051 | _mov(T, T_fp); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4052 | if (DestTy != IceType_i32) { |
| 4053 | Variable *T_1 = makeReg(DestTy); |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4054 | lowerCast(InstCast::create(Func, InstCast::Trunc, T_1, T)); |
| 4055 | T = T_1; |
| 4056 | } |
| 4057 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4058 | break; |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4059 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4060 | case InstCast::Sitofp: |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4061 | case InstCast::Uitofp: { |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4062 | const bool SourceIsSigned = CastKind == InstCast::Sitofp; |
| 4063 | |
| 4064 | if (isVectorType(DestTy)) { |
| 4065 | assert(typeElementType(DestTy) == IceType_f32); |
| 4066 | auto *T = makeReg(DestTy); |
| 4067 | Variable *Src0R = legalizeToReg(Src0); |
| 4068 | _vcvt(T, Src0R, |
| 4069 | SourceIsSigned ? InstARM32Vcvt::Vsi2s : InstARM32Vcvt::Vui2s); |
| 4070 | _mov(Dest, T); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4071 | break; |
| 4072 | } |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4073 | |
| 4074 | const bool DestIsF32 = isFloat32Asserting32Or64(DestTy); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4075 | if (Src0->getType() == IceType_i64) { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 4076 | llvm::report_fatal_error("i64-to-fp should have been pre-lowered."); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4077 | } |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4078 | // sitofp: |
| 4079 | // t1.i32 = sext src.int @ sign-extends src0 if needed. |
| 4080 | // t2.fp32 = vmov t1.i32 |
| 4081 | // t3.fp = vcvt.{fp}.s32 @ fp is either f32 or f64 |
| 4082 | // uitofp: |
| 4083 | // t1.i32 = zext src.int @ zero-extends src0 if needed. |
| 4084 | // t2.fp32 = vmov t1.i32 |
| 4085 | // t3.fp = vcvt.{fp}.s32 @ fp is either f32 or f64 |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4086 | if (Src0->getType() != IceType_i32) { |
| 4087 | Variable *Src0R_32 = makeReg(IceType_i32); |
| 4088 | lowerCast(InstCast::create(Func, SourceIsSigned ? InstCast::Sext |
| 4089 | : InstCast::Zext, |
| 4090 | Src0R_32, Src0)); |
| 4091 | Src0 = Src0R_32; |
| 4092 | } |
| 4093 | Variable *Src0R = legalizeToReg(Src0); |
| 4094 | Variable *Src0R_f32 = makeReg(IceType_f32); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4095 | _mov(Src0R_f32, Src0R); |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4096 | Src0R = Src0R_f32; |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4097 | Variable *T = makeReg(DestTy); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4098 | const InstARM32Vcvt::VcvtVariant Conversion = |
| 4099 | DestIsF32 |
| 4100 | ? (SourceIsSigned ? InstARM32Vcvt::Si2s : InstARM32Vcvt::Ui2s) |
| 4101 | : (SourceIsSigned ? InstARM32Vcvt::Si2d : InstARM32Vcvt::Ui2d); |
| 4102 | _vcvt(T, Src0R, Conversion); |
John Porto | c31e2ed | 2015-09-11 05:17:08 -0700 | [diff] [blame] | 4103 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4104 | break; |
| 4105 | } |
| 4106 | case InstCast::Bitcast: { |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 4107 | Operand *Src0 = Instr->getSrc(0); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4108 | if (DestTy == Src0->getType()) { |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 4109 | auto *Assign = InstAssign::create(Func, Dest, Src0); |
Jan Voung | 66c3d5e | 2015-06-04 17:02:31 -0700 | [diff] [blame] | 4110 | lowerAssign(Assign); |
| 4111 | return; |
| 4112 | } |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4113 | switch (DestTy) { |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4114 | case IceType_NUM: |
| 4115 | case IceType_void: |
| 4116 | llvm::report_fatal_error("Unexpected bitcast."); |
| 4117 | case IceType_i1: |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 4118 | UnimplementedLoweringError(this, Instr); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4119 | break; |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4120 | case IceType_i8: |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 4121 | assert(Src0->getType() == IceType_v8i1); |
| 4122 | llvm::report_fatal_error( |
| 4123 | "i8 to v8i1 conversion should have been prelowered."); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4124 | break; |
| 4125 | case IceType_i16: |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 4126 | assert(Src0->getType() == IceType_v16i1); |
| 4127 | llvm::report_fatal_error( |
| 4128 | "i16 to v16i1 conversion should have been prelowered."); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4129 | break; |
| 4130 | case IceType_i32: |
| 4131 | case IceType_f32: { |
| 4132 | Variable *Src0R = legalizeToReg(Src0); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4133 | Variable *T = makeReg(DestTy); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4134 | _mov(T, Src0R); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4135 | lowerAssign(InstAssign::create(Func, Dest, T)); |
| 4136 | break; |
| 4137 | } |
| 4138 | case IceType_i64: { |
| 4139 | // t0, t1 <- src0 |
| 4140 | // dest[31..0] = t0 |
| 4141 | // dest[63..32] = t1 |
| 4142 | assert(Src0->getType() == IceType_f64); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4143 | auto *T = llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64)); |
| 4144 | T->initHiLo(Func); |
John Porto | e0b829f | 2015-09-28 09:50:48 -0700 | [diff] [blame] | 4145 | configureBitcastTemporary(T); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4146 | Variable *Src0R = legalizeToReg(Src0); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4147 | _mov(T, Src0R); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4148 | Context.insert<InstFakeUse>(T->getHi()); |
| 4149 | Context.insert<InstFakeUse>(T->getLo()); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4150 | lowerAssign(InstAssign::create(Func, Dest, T)); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4151 | break; |
| 4152 | } |
| 4153 | case IceType_f64: { |
| 4154 | // T0 <- lo(src) |
| 4155 | // T1 <- hi(src) |
| 4156 | // vmov T2, T0, T1 |
| 4157 | // Dest <- T2 |
| 4158 | assert(Src0->getType() == IceType_i64); |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4159 | Variable *T = makeReg(DestTy); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4160 | auto *Src64 = llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64)); |
| 4161 | Src64->initHiLo(Func); |
John Porto | e0b829f | 2015-09-28 09:50:48 -0700 | [diff] [blame] | 4162 | configureBitcastTemporary(Src64); |
| 4163 | lowerAssign(InstAssign::create(Func, Src64, Src0)); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4164 | _mov(T, Src64); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4165 | lowerAssign(InstAssign::create(Func, Dest, T)); |
| 4166 | break; |
| 4167 | } |
| 4168 | case IceType_v8i1: |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 4169 | assert(Src0->getType() == IceType_i8); |
| 4170 | llvm::report_fatal_error( |
| 4171 | "v8i1 to i8 conversion should have been prelowered."); |
| 4172 | break; |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4173 | case IceType_v16i1: |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 4174 | assert(Src0->getType() == IceType_i16); |
| 4175 | llvm::report_fatal_error( |
| 4176 | "v16i1 to i16 conversion should have been prelowered."); |
| 4177 | break; |
| 4178 | case IceType_v4i1: |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4179 | case IceType_v8i16: |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4180 | case IceType_v16i8: |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4181 | case IceType_v4f32: |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4182 | case IceType_v4i32: { |
John Porto | e88c7de | 2016-04-14 11:51:38 -0700 | [diff] [blame] | 4183 | assert(typeWidthInBytes(DestTy) == typeWidthInBytes(Src0->getType())); |
| 4184 | assert(isVectorType(DestTy) == isVectorType(Src0->getType())); |
| 4185 | Variable *T = makeReg(DestTy); |
John Porto | 7e6aa5a | 2016-03-02 15:10:19 -0800 | [diff] [blame] | 4186 | _mov(T, Src0); |
| 4187 | _mov(Dest, T); |
John Porto | f977f71 | 2015-09-14 16:28:33 -0700 | [diff] [blame] | 4188 | break; |
| 4189 | } |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 4190 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4191 | break; |
| 4192 | } |
| 4193 | } |
| 4194 | } |
| 4195 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 4196 | void TargetARM32::lowerExtractElement(const InstExtractElement *Instr) { |
Eric Holk | 658bae2 | 2016-02-08 15:22:18 -0800 | [diff] [blame] | 4197 | Variable *Dest = Instr->getDest(); |
| 4198 | Type DestTy = Dest->getType(); |
| 4199 | |
| 4200 | Variable *Src0 = legalizeToReg(Instr->getSrc(0)); |
| 4201 | Operand *Src1 = Instr->getSrc(1); |
| 4202 | |
| 4203 | if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src1)) { |
| 4204 | const uint32_t Index = Imm->getValue(); |
| 4205 | Variable *T = makeReg(DestTy); |
| 4206 | Variable *TSrc0 = makeReg(Src0->getType()); |
| 4207 | |
| 4208 | if (isFloatingType(DestTy)) { |
| 4209 | // We need to make sure the source is in a suitable register. |
| 4210 | TSrc0->setRegClass(RegARM32::RCARM32_QtoS); |
| 4211 | } |
| 4212 | |
| 4213 | _mov(TSrc0, Src0); |
| 4214 | _extractelement(T, TSrc0, Index); |
| 4215 | _mov(Dest, T); |
| 4216 | return; |
| 4217 | } |
| 4218 | assert(false && "extractelement requires a constant index"); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4219 | } |
| 4220 | |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4221 | namespace { |
| 4222 | // Validates FCMPARM32_TABLE's declaration w.r.t. InstFcmp::FCondition ordering |
| 4223 | // (and naming). |
| 4224 | enum { |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 4225 | #define X(val, CC0, CC1, CC0_V, CC1_V, INV_V, NEG_V) _fcmp_ll_##val, |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4226 | FCMPARM32_TABLE |
| 4227 | #undef X |
| 4228 | _fcmp_ll_NUM |
| 4229 | }; |
| 4230 | |
| 4231 | enum { |
| 4232 | #define X(tag, str) _fcmp_hl_##tag = InstFcmp::tag, |
| 4233 | ICEINSTFCMP_TABLE |
| 4234 | #undef X |
| 4235 | _fcmp_hl_NUM |
| 4236 | }; |
| 4237 | |
Jim Stichnoth | b0051df | 2016-01-13 11:39:15 -0800 | [diff] [blame] | 4238 | static_assert((uint32_t)_fcmp_hl_NUM == (uint32_t)_fcmp_ll_NUM, |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4239 | "Inconsistency between high-level and low-level fcmp tags."); |
| 4240 | #define X(tag, str) \ |
| 4241 | static_assert( \ |
Jim Stichnoth | b0051df | 2016-01-13 11:39:15 -0800 | [diff] [blame] | 4242 | (uint32_t)_fcmp_hl_##tag == (uint32_t)_fcmp_ll_##tag, \ |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4243 | "Inconsistency between high-level and low-level fcmp tag " #tag); |
| 4244 | ICEINSTFCMP_TABLE |
| 4245 | #undef X |
| 4246 | |
| 4247 | struct { |
| 4248 | CondARM32::Cond CC0; |
| 4249 | CondARM32::Cond CC1; |
| 4250 | } TableFcmp[] = { |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 4251 | #define X(val, CC0, CC1, CC0_V, CC1_V, INV_V, NEG_V) \ |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4252 | { CondARM32::CC0, CondARM32::CC1 } \ |
| 4253 | , |
| 4254 | FCMPARM32_TABLE |
| 4255 | #undef X |
| 4256 | }; |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4257 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 4258 | bool isFloatingPointZero(const Operand *Src) { |
| 4259 | if (const auto *F32 = llvm::dyn_cast<const ConstantFloat>(Src)) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4260 | return Utils::isPositiveZero(F32->getValue()); |
| 4261 | } |
| 4262 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 4263 | if (const auto *F64 = llvm::dyn_cast<const ConstantDouble>(Src)) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4264 | return Utils::isPositiveZero(F64->getValue()); |
| 4265 | } |
| 4266 | |
| 4267 | return false; |
| 4268 | } |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4269 | } // end of anonymous namespace |
| 4270 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4271 | TargetARM32::CondWhenTrue TargetARM32::lowerFcmpCond(const InstFcmp *Instr) { |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4272 | InstFcmp::FCond Condition = Instr->getCondition(); |
| 4273 | switch (Condition) { |
| 4274 | case InstFcmp::False: |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4275 | return CondWhenTrue(CondARM32::kNone); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4276 | case InstFcmp::True: |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4277 | return CondWhenTrue(CondARM32::AL); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4278 | break; |
| 4279 | default: { |
| 4280 | Variable *Src0R = legalizeToReg(Instr->getSrc(0)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4281 | Operand *Src1 = Instr->getSrc(1); |
| 4282 | if (isFloatingPointZero(Src1)) { |
| 4283 | _vcmp(Src0R, OperandARM32FlexFpZero::create(Func, Src0R->getType())); |
| 4284 | } else { |
| 4285 | _vcmp(Src0R, legalizeToReg(Src1)); |
| 4286 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4287 | _vmrs(); |
| 4288 | assert(Condition < llvm::array_lengthof(TableFcmp)); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4289 | return CondWhenTrue(TableFcmp[Condition].CC0, TableFcmp[Condition].CC1); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4290 | } |
| 4291 | } |
| 4292 | } |
| 4293 | |
| 4294 | void TargetARM32::lowerFcmp(const InstFcmp *Instr) { |
| 4295 | Variable *Dest = Instr->getDest(); |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 4296 | const Type DestTy = Dest->getType(); |
| 4297 | |
| 4298 | if (isVectorType(DestTy)) { |
| 4299 | if (Instr->getCondition() == InstFcmp::False) { |
| 4300 | constexpr Type SafeTypeForMovingConstant = IceType_v4i32; |
| 4301 | auto *T = makeReg(SafeTypeForMovingConstant); |
| 4302 | _mov(T, llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0))); |
| 4303 | _mov(Dest, T); |
| 4304 | return; |
| 4305 | } |
| 4306 | |
| 4307 | if (Instr->getCondition() == InstFcmp::True) { |
| 4308 | constexpr Type SafeTypeForMovingConstant = IceType_v4i32; |
| 4309 | auto *T = makeReg(SafeTypeForMovingConstant); |
| 4310 | _mov(T, llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(1))); |
| 4311 | _mov(Dest, T); |
| 4312 | return; |
| 4313 | } |
| 4314 | |
| 4315 | Variable *T0; |
| 4316 | Variable *T1; |
| 4317 | bool Negate = false; |
| 4318 | auto *Src0 = legalizeToReg(Instr->getSrc(0)); |
| 4319 | auto *Src1 = legalizeToReg(Instr->getSrc(1)); |
| 4320 | |
| 4321 | switch (Instr->getCondition()) { |
| 4322 | default: |
| 4323 | llvm::report_fatal_error("Unhandled fp comparison."); |
| 4324 | #define _Vcnone(Tptr, S0, S1) \ |
| 4325 | do { \ |
| 4326 | *(Tptr) = nullptr; \ |
| 4327 | } while (0) |
| 4328 | #define _Vceq(Tptr, S0, S1) \ |
| 4329 | do { \ |
| 4330 | *(Tptr) = makeReg(DestTy); \ |
| 4331 | _vceq(*(Tptr), S0, S1); \ |
| 4332 | } while (0) |
| 4333 | #define _Vcge(Tptr, S0, S1) \ |
| 4334 | do { \ |
| 4335 | *(Tptr) = makeReg(DestTy); \ |
| 4336 | _vcge(*(Tptr), S0, S1)->setSignType(InstARM32::FS_Signed); \ |
| 4337 | } while (0) |
| 4338 | #define _Vcgt(Tptr, S0, S1) \ |
| 4339 | do { \ |
| 4340 | *(Tptr) = makeReg(DestTy); \ |
| 4341 | _vcgt(*(Tptr), S0, S1)->setSignType(InstARM32::FS_Signed); \ |
| 4342 | } while (0) |
| 4343 | #define X(val, CC0, CC1, CC0_V, CC1_V, INV_V, NEG_V) \ |
| 4344 | case InstFcmp::val: { \ |
| 4345 | _Vc##CC0_V(&T0, (INV_V) ? Src1 : Src0, (INV_V) ? Src0 : Src1); \ |
| 4346 | _Vc##CC1_V(&T1, (INV_V) ? Src0 : Src1, (INV_V) ? Src1 : Src0); \ |
| 4347 | Negate = NEG_V; \ |
| 4348 | } break; |
| 4349 | FCMPARM32_TABLE |
| 4350 | #undef X |
| 4351 | #undef _Vcgt |
| 4352 | #undef _Vcge |
| 4353 | #undef _Vceq |
| 4354 | #undef _Vcnone |
| 4355 | } |
| 4356 | assert(T0 != nullptr); |
| 4357 | Variable *T = T0; |
| 4358 | if (T1 != nullptr) { |
| 4359 | T = makeReg(DestTy); |
| 4360 | _vorr(T, T0, T1); |
| 4361 | } |
| 4362 | |
| 4363 | if (Negate) { |
| 4364 | auto *TNeg = makeReg(DestTy); |
| 4365 | _vmvn(TNeg, T); |
| 4366 | T = TNeg; |
| 4367 | } |
| 4368 | |
| 4369 | _mov(Dest, T); |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4370 | return; |
| 4371 | } |
| 4372 | |
Karl Schimpf | b9f2722 | 2015-11-09 12:09:58 -0800 | [diff] [blame] | 4373 | Variable *T = makeReg(IceType_i1); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4374 | Operand *_1 = legalize(Ctx->getConstantInt32(1), Legal_Reg | Legal_Flex); |
| 4375 | Operand *_0 = |
| 4376 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4377 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4378 | CondWhenTrue Cond = lowerFcmpCond(Instr); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4379 | |
| 4380 | bool RedefineT = false; |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4381 | if (Cond.WhenTrue0 != CondARM32::AL) { |
| 4382 | _mov(T, _0); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4383 | RedefineT = true; |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4384 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4385 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4386 | if (Cond.WhenTrue0 == CondARM32::kNone) { |
| 4387 | _mov(Dest, T); |
| 4388 | return; |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4389 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4390 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4391 | if (RedefineT) { |
| 4392 | _mov_redefined(T, _1, Cond.WhenTrue0); |
| 4393 | } else { |
| 4394 | _mov(T, _1, Cond.WhenTrue0); |
| 4395 | } |
| 4396 | |
| 4397 | if (Cond.WhenTrue1 != CondARM32::kNone) { |
| 4398 | _mov_redefined(T, _1, Cond.WhenTrue1); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4399 | } |
| 4400 | |
John Porto | 2f5534f | 2015-09-18 15:59:47 -0700 | [diff] [blame] | 4401 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4402 | } |
| 4403 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4404 | TargetARM32::CondWhenTrue |
| 4405 | TargetARM32::lowerInt64IcmpCond(InstIcmp::ICond Condition, Operand *Src0, |
| 4406 | Operand *Src1) { |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4407 | assert(Condition < llvm::array_lengthof(TableIcmp64)); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4408 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4409 | Int32Operands SrcsLo(loOperand(Src0), loOperand(Src1)); |
| 4410 | Int32Operands SrcsHi(hiOperand(Src0), hiOperand(Src1)); |
| 4411 | assert(SrcsLo.hasConstOperand() == SrcsHi.hasConstOperand()); |
| 4412 | assert(SrcsLo.swappedOperands() == SrcsHi.swappedOperands()); |
| 4413 | |
| 4414 | if (SrcsLo.hasConstOperand()) { |
| 4415 | const uint32_t ValueLo = SrcsLo.getConstantValue(); |
| 4416 | const uint32_t ValueHi = SrcsHi.getConstantValue(); |
| 4417 | const uint64_t Value = (static_cast<uint64_t>(ValueHi) << 32) | ValueLo; |
| 4418 | if ((Condition == InstIcmp::Eq || Condition == InstIcmp::Ne) && |
| 4419 | Value == 0) { |
| 4420 | Variable *T = makeReg(IceType_i32); |
| 4421 | Variable *Src0LoR = SrcsLo.src0R(this); |
| 4422 | Variable *Src0HiR = SrcsHi.src0R(this); |
| 4423 | _orrs(T, Src0LoR, Src0HiR); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4424 | Context.insert<InstFakeUse>(T); |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4425 | return CondWhenTrue(TableIcmp64[Condition].C1); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4426 | } |
| 4427 | |
| 4428 | Variable *Src0RLo = SrcsLo.src0R(this); |
| 4429 | Variable *Src0RHi = SrcsHi.src0R(this); |
| 4430 | Operand *Src1RFLo = SrcsLo.src1RF(this); |
| 4431 | Operand *Src1RFHi = ValueLo == ValueHi ? Src1RFLo : SrcsHi.src1RF(this); |
| 4432 | |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4433 | const bool UseRsb = |
| 4434 | TableIcmp64[Condition].Swapped != SrcsLo.swappedOperands(); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4435 | |
| 4436 | if (UseRsb) { |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4437 | if (TableIcmp64[Condition].IsSigned) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4438 | Variable *T = makeReg(IceType_i32); |
| 4439 | _rsbs(T, Src0RLo, Src1RFLo); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4440 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4441 | |
| 4442 | T = makeReg(IceType_i32); |
| 4443 | _rscs(T, Src0RHi, Src1RFHi); |
| 4444 | // We need to add a FakeUse here because liveness gets mad at us (Def |
| 4445 | // without Use.) Note that flag-setting instructions are considered to |
| 4446 | // have side effects and, therefore, are not DCE'ed. |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4447 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4448 | } else { |
| 4449 | Variable *T = makeReg(IceType_i32); |
| 4450 | _rsbs(T, Src0RHi, Src1RFHi); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4451 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4452 | |
| 4453 | T = makeReg(IceType_i32); |
| 4454 | _rsbs(T, Src0RLo, Src1RFLo, CondARM32::EQ); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4455 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4456 | } |
| 4457 | } else { |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4458 | if (TableIcmp64[Condition].IsSigned) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4459 | _cmp(Src0RLo, Src1RFLo); |
| 4460 | Variable *T = makeReg(IceType_i32); |
| 4461 | _sbcs(T, Src0RHi, Src1RFHi); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4462 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4463 | } else { |
| 4464 | _cmp(Src0RHi, Src1RFHi); |
| 4465 | _cmp(Src0RLo, Src1RFLo, CondARM32::EQ); |
| 4466 | } |
| 4467 | } |
| 4468 | |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4469 | return CondWhenTrue(TableIcmp64[Condition].C1); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4470 | } |
| 4471 | |
| 4472 | Variable *Src0RLo, *Src0RHi; |
| 4473 | Operand *Src1RFLo, *Src1RFHi; |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4474 | if (TableIcmp64[Condition].Swapped) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4475 | Src0RLo = legalizeToReg(loOperand(Src1)); |
| 4476 | Src0RHi = legalizeToReg(hiOperand(Src1)); |
| 4477 | Src1RFLo = legalizeToReg(loOperand(Src0)); |
| 4478 | Src1RFHi = legalizeToReg(hiOperand(Src0)); |
| 4479 | } else { |
| 4480 | Src0RLo = legalizeToReg(loOperand(Src0)); |
| 4481 | Src0RHi = legalizeToReg(hiOperand(Src0)); |
| 4482 | Src1RFLo = legalizeToReg(loOperand(Src1)); |
| 4483 | Src1RFHi = legalizeToReg(hiOperand(Src1)); |
| 4484 | } |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4485 | |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4486 | // a=icmp cond, b, c ==> |
| 4487 | // GCC does: |
| 4488 | // cmp b.hi, c.hi or cmp b.lo, c.lo |
| 4489 | // cmp.eq b.lo, c.lo sbcs t1, b.hi, c.hi |
| 4490 | // mov.<C1> t, #1 mov.<C1> t, #1 |
| 4491 | // mov.<C2> t, #0 mov.<C2> t, #0 |
| 4492 | // mov a, t mov a, t |
| 4493 | // where the "cmp.eq b.lo, c.lo" is used for unsigned and "sbcs t1, hi, hi" |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 4494 | // is used for signed compares. In some cases, b and c need to be swapped as |
| 4495 | // well. |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4496 | // |
| 4497 | // LLVM does: |
| 4498 | // for EQ and NE: |
| 4499 | // eor t1, b.hi, c.hi |
| 4500 | // eor t2, b.lo, c.hi |
| 4501 | // orrs t, t1, t2 |
| 4502 | // mov.<C> t, #1 |
| 4503 | // mov a, t |
| 4504 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 4505 | // that's nice in that it's just as short but has fewer dependencies for |
| 4506 | // better ILP at the cost of more registers. |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4507 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 4508 | // Otherwise for signed/unsigned <, <=, etc. LLVM uses a sequence with two |
| 4509 | // unconditional mov #0, two cmps, two conditional mov #1, and one |
| 4510 | // conditional reg mov. That has few dependencies for good ILP, but is a |
| 4511 | // longer sequence. |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4512 | // |
| 4513 | // So, we are going with the GCC version since it's usually better (except |
| 4514 | // perhaps for eq/ne). We could revisit special-casing eq/ne later. |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4515 | if (TableIcmp64[Condition].IsSigned) { |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4516 | Variable *ScratchReg = makeReg(IceType_i32); |
| 4517 | _cmp(Src0RLo, Src1RFLo); |
| 4518 | _sbcs(ScratchReg, Src0RHi, Src1RFHi); |
| 4519 | // ScratchReg isn't going to be used, but we need the side-effect of |
| 4520 | // setting flags from this operation. |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4521 | Context.insert<InstFakeUse>(ScratchReg); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4522 | } else { |
| 4523 | _cmp(Src0RHi, Src1RFHi); |
| 4524 | _cmp(Src0RLo, Src1RFLo, CondARM32::EQ); |
| 4525 | } |
Jim Stichnoth | 2d6c826 | 2016-02-07 09:50:27 -0800 | [diff] [blame] | 4526 | return CondWhenTrue(TableIcmp64[Condition].C1); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4527 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4528 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4529 | TargetARM32::CondWhenTrue |
| 4530 | TargetARM32::lowerInt32IcmpCond(InstIcmp::ICond Condition, Operand *Src0, |
| 4531 | Operand *Src1) { |
| 4532 | Int32Operands Srcs(Src0, Src1); |
| 4533 | if (!Srcs.hasConstOperand()) { |
| 4534 | |
| 4535 | Variable *Src0R = Srcs.src0R(this); |
| 4536 | Operand *Src1RF = Srcs.src1RF(this); |
| 4537 | _cmp(Src0R, Src1RF); |
| 4538 | return CondWhenTrue(getIcmp32Mapping(Condition)); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4539 | } |
| 4540 | |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4541 | Variable *Src0R = Srcs.src0R(this); |
| 4542 | const int32_t Value = Srcs.getConstantValue(); |
| 4543 | if ((Condition == InstIcmp::Eq || Condition == InstIcmp::Ne) && Value == 0) { |
| 4544 | _tst(Src0R, Src0R); |
| 4545 | return CondWhenTrue(getIcmp32Mapping(Condition)); |
| 4546 | } |
| 4547 | |
| 4548 | if (!Srcs.swappedOperands() && !Srcs.immediateIsFlexEncodable() && |
| 4549 | Srcs.negatedImmediateIsFlexEncodable()) { |
| 4550 | Operand *Src1F = Srcs.negatedSrc1F(this); |
| 4551 | _cmn(Src0R, Src1F); |
| 4552 | return CondWhenTrue(getIcmp32Mapping(Condition)); |
| 4553 | } |
| 4554 | |
| 4555 | Operand *Src1RF = Srcs.src1RF(this); |
| 4556 | if (!Srcs.swappedOperands()) { |
| 4557 | _cmp(Src0R, Src1RF); |
| 4558 | } else { |
| 4559 | Variable *T = makeReg(IceType_i32); |
| 4560 | _rsbs(T, Src0R, Src1RF); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4561 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4562 | } |
| 4563 | return CondWhenTrue(getIcmp32Mapping(Condition)); |
| 4564 | } |
| 4565 | |
| 4566 | TargetARM32::CondWhenTrue |
| 4567 | TargetARM32::lowerInt8AndInt16IcmpCond(InstIcmp::ICond Condition, Operand *Src0, |
| 4568 | Operand *Src1) { |
| 4569 | Int32Operands Srcs(Src0, Src1); |
| 4570 | const int32_t ShAmt = 32 - getScalarIntBitWidth(Src0->getType()); |
| 4571 | assert(ShAmt >= 0); |
| 4572 | |
| 4573 | if (!Srcs.hasConstOperand()) { |
| 4574 | Variable *Src0R = makeReg(IceType_i32); |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 4575 | Operand *ShAmtImm = shAmtImm(ShAmt); |
| 4576 | _lsl(Src0R, legalizeToReg(Src0), ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4577 | |
| 4578 | Variable *Src1R = legalizeToReg(Src1); |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 4579 | auto *Src1F = OperandARM32FlexReg::create(Func, IceType_i32, Src1R, |
| 4580 | OperandARM32::LSL, ShAmtImm); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4581 | _cmp(Src0R, Src1F); |
| 4582 | return CondWhenTrue(getIcmp32Mapping(Condition)); |
| 4583 | } |
| 4584 | |
| 4585 | const int32_t Value = Srcs.getConstantValue(); |
| 4586 | if ((Condition == InstIcmp::Eq || Condition == InstIcmp::Ne) && Value == 0) { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 4587 | Operand *ShAmtImm = shAmtImm(ShAmt); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4588 | Variable *T = makeReg(IceType_i32); |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 4589 | _lsls(T, Srcs.src0R(this), ShAmtImm); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4590 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4591 | return CondWhenTrue(getIcmp32Mapping(Condition)); |
| 4592 | } |
| 4593 | |
| 4594 | Variable *ConstR = makeReg(IceType_i32); |
| 4595 | _mov(ConstR, |
| 4596 | legalize(Ctx->getConstantInt32(Value << ShAmt), Legal_Reg | Legal_Flex)); |
| 4597 | Operand *NonConstF = OperandARM32FlexReg::create( |
| 4598 | Func, IceType_i32, Srcs.src0R(this), OperandARM32::LSL, |
| 4599 | Ctx->getConstantInt32(ShAmt)); |
| 4600 | |
| 4601 | if (Srcs.swappedOperands()) { |
| 4602 | _cmp(ConstR, NonConstF); |
| 4603 | } else { |
| 4604 | Variable *T = makeReg(IceType_i32); |
| 4605 | _rsbs(T, ConstR, NonConstF); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 4606 | Context.insert<InstFakeUse>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4607 | } |
| 4608 | return CondWhenTrue(getIcmp32Mapping(Condition)); |
| 4609 | } |
| 4610 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 4611 | TargetARM32::CondWhenTrue TargetARM32::lowerIcmpCond(const InstIcmp *Instr) { |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4612 | return lowerIcmpCond(Instr->getCondition(), Instr->getSrc(0), |
| 4613 | Instr->getSrc(1)); |
| 4614 | } |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4615 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4616 | TargetARM32::CondWhenTrue TargetARM32::lowerIcmpCond(InstIcmp::ICond Condition, |
| 4617 | Operand *Src0, |
| 4618 | Operand *Src1) { |
| 4619 | Src0 = legalizeUndef(Src0); |
| 4620 | Src1 = legalizeUndef(Src1); |
| 4621 | |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4622 | // a=icmp cond b, c ==> |
| 4623 | // GCC does: |
| 4624 | // <u/s>xtb tb, b |
| 4625 | // <u/s>xtb tc, c |
| 4626 | // cmp tb, tc |
| 4627 | // mov.C1 t, #0 |
| 4628 | // mov.C2 t, #1 |
| 4629 | // mov a, t |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 4630 | // where the unsigned/sign extension is not needed for 32-bit. They also have |
| 4631 | // special cases for EQ and NE. E.g., for NE: |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4632 | // <extend to tb, tc> |
| 4633 | // subs t, tb, tc |
| 4634 | // movne t, #1 |
| 4635 | // mov a, t |
| 4636 | // |
| 4637 | // LLVM does: |
| 4638 | // lsl tb, b, #<N> |
| 4639 | // mov t, #0 |
| 4640 | // cmp tb, c, lsl #<N> |
| 4641 | // mov.<C> t, #1 |
| 4642 | // mov a, t |
| 4643 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 4644 | // the left shift is by 0, 16, or 24, which allows the comparison to focus on |
| 4645 | // the digits that actually matter (for 16-bit or 8-bit signed/unsigned). For |
| 4646 | // the unsigned case, for some reason it does similar to GCC and does a uxtb |
| 4647 | // first. It's not clear to me why that special-casing is needed. |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4648 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 4649 | // We'll go with the LLVM way for now, since it's shorter and has just as few |
| 4650 | // dependencies. |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4651 | switch (Src0->getType()) { |
| 4652 | default: |
| 4653 | llvm::report_fatal_error("Unhandled type in lowerIcmpCond"); |
Eric Holk | cc69fa2 | 2016-02-10 13:07:06 -0800 | [diff] [blame] | 4654 | case IceType_i1: |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 4655 | case IceType_i8: |
| 4656 | case IceType_i16: |
| 4657 | return lowerInt8AndInt16IcmpCond(Condition, Src0, Src1); |
| 4658 | case IceType_i32: |
| 4659 | return lowerInt32IcmpCond(Condition, Src0, Src1); |
| 4660 | case IceType_i64: |
| 4661 | return lowerInt64IcmpCond(Condition, Src0, Src1); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4662 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4663 | } |
| 4664 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 4665 | void TargetARM32::lowerIcmp(const InstIcmp *Instr) { |
| 4666 | Variable *Dest = Instr->getDest(); |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 4667 | const Type DestTy = Dest->getType(); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4668 | |
John Porto | a4d100a | 2016-04-18 15:32:27 -0700 | [diff] [blame] | 4669 | if (isVectorType(DestTy)) { |
| 4670 | auto *T = makeReg(DestTy); |
| 4671 | auto *Src0 = legalizeToReg(Instr->getSrc(0)); |
| 4672 | auto *Src1 = legalizeToReg(Instr->getSrc(1)); |
| 4673 | const Type SrcTy = Src0->getType(); |
| 4674 | |
| 4675 | bool NeedsShl = false; |
| 4676 | Type NewTypeAfterShl; |
| 4677 | SizeT ShAmt; |
| 4678 | switch (SrcTy) { |
| 4679 | default: |
| 4680 | break; |
| 4681 | case IceType_v16i1: |
| 4682 | NeedsShl = true; |
| 4683 | NewTypeAfterShl = IceType_v16i8; |
| 4684 | ShAmt = 7; |
| 4685 | break; |
| 4686 | case IceType_v8i1: |
| 4687 | NeedsShl = true; |
| 4688 | NewTypeAfterShl = IceType_v8i16; |
| 4689 | ShAmt = 15; |
| 4690 | break; |
| 4691 | case IceType_v4i1: |
| 4692 | NeedsShl = true; |
| 4693 | NewTypeAfterShl = IceType_v4i32; |
| 4694 | ShAmt = 31; |
| 4695 | break; |
| 4696 | } |
| 4697 | |
| 4698 | if (NeedsShl) { |
| 4699 | auto *Imm = llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(ShAmt)); |
| 4700 | auto *Src0T = makeReg(NewTypeAfterShl); |
| 4701 | auto *Src0Shl = makeReg(NewTypeAfterShl); |
| 4702 | _mov(Src0T, Src0); |
| 4703 | _vshl(Src0Shl, Src0T, Imm); |
| 4704 | Src0 = Src0Shl; |
| 4705 | |
| 4706 | auto *Src1T = makeReg(NewTypeAfterShl); |
| 4707 | auto *Src1Shl = makeReg(NewTypeAfterShl); |
| 4708 | _mov(Src1T, Src1); |
| 4709 | _vshl(Src1Shl, Src1T, Imm); |
| 4710 | Src1 = Src1Shl; |
| 4711 | } |
| 4712 | |
| 4713 | switch (Instr->getCondition()) { |
| 4714 | default: |
| 4715 | llvm::report_fatal_error("Unhandled integer comparison."); |
| 4716 | #define _Vceq(T, S0, S1, Signed) _vceq(T, S0, S1) |
| 4717 | #define _Vcge(T, S0, S1, Signed) \ |
| 4718 | _vcge(T, S0, S1) \ |
| 4719 | ->setSignType(Signed ? InstARM32::FS_Signed : InstARM32::FS_Unsigned) |
| 4720 | #define _Vcgt(T, S0, S1, Signed) \ |
| 4721 | _vcgt(T, S0, S1) \ |
| 4722 | ->setSignType(Signed ? InstARM32::FS_Signed : InstARM32::FS_Unsigned) |
| 4723 | #define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V) \ |
| 4724 | case InstIcmp::val: { \ |
| 4725 | _Vc##C_V(T, (INV_V) ? Src1 : Src0, (INV_V) ? Src0 : Src1, is_signed); \ |
| 4726 | if (NEG_V) { \ |
| 4727 | auto *TInv = makeReg(DestTy); \ |
| 4728 | _vmvn(TInv, T); \ |
| 4729 | T = TInv; \ |
| 4730 | } \ |
| 4731 | } break; |
| 4732 | ICMPARM32_TABLE |
| 4733 | #undef X |
| 4734 | #undef _Vcgt |
| 4735 | #undef _Vcge |
| 4736 | #undef _Vceq |
| 4737 | } |
| 4738 | _mov(Dest, T); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4739 | return; |
| 4740 | } |
| 4741 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4742 | Operand *_0 = |
| 4743 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
| 4744 | Operand *_1 = legalize(Ctx->getConstantInt32(1), Legal_Reg | Legal_Flex); |
Karl Schimpf | b9f2722 | 2015-11-09 12:09:58 -0800 | [diff] [blame] | 4745 | Variable *T = makeReg(IceType_i1); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4746 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4747 | _mov(T, _0); |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 4748 | CondWhenTrue Cond = lowerIcmpCond(Instr); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4749 | _mov_redefined(T, _1, Cond.WhenTrue0); |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4750 | _mov(Dest, T); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 4751 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 4752 | assert(Cond.WhenTrue1 == CondARM32::kNone); |
| 4753 | |
Jan Voung | 3bfd99a | 2015-05-22 16:35:25 -0700 | [diff] [blame] | 4754 | return; |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4755 | } |
| 4756 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 4757 | void TargetARM32::lowerInsertElement(const InstInsertElement *Instr) { |
Eric Holk | 658bae2 | 2016-02-08 15:22:18 -0800 | [diff] [blame] | 4758 | Variable *Dest = Instr->getDest(); |
| 4759 | Type DestTy = Dest->getType(); |
| 4760 | |
| 4761 | Variable *Src0 = legalizeToReg(Instr->getSrc(0)); |
| 4762 | Variable *Src1 = legalizeToReg(Instr->getSrc(1)); |
| 4763 | Operand *Src2 = Instr->getSrc(2); |
| 4764 | |
| 4765 | if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src2)) { |
| 4766 | const uint32_t Index = Imm->getValue(); |
| 4767 | Variable *T = makeReg(DestTy); |
| 4768 | |
| 4769 | if (isFloatingType(DestTy)) { |
| 4770 | T->setRegClass(RegARM32::RCARM32_QtoS); |
| 4771 | } |
| 4772 | |
| 4773 | _mov(T, Src0); |
| 4774 | _insertelement(T, Src1, Index); |
| 4775 | _set_dest_redefined(); |
| 4776 | _mov(Dest, T); |
| 4777 | return; |
| 4778 | } |
| 4779 | assert(false && "insertelement requires a constant index"); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4780 | } |
| 4781 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4782 | namespace { |
| 4783 | inline uint64_t getConstantMemoryOrder(Operand *Opnd) { |
Jim Stichnoth | 5bff61c | 2015-10-28 09:26:00 -0700 | [diff] [blame] | 4784 | if (auto *Integer = llvm::dyn_cast<ConstantInteger32>(Opnd)) |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4785 | return Integer->getValue(); |
| 4786 | return Intrinsics::MemoryOrderInvalid; |
| 4787 | } |
| 4788 | } // end of anonymous namespace |
| 4789 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4790 | void TargetARM32::lowerLoadLinkedStoreExclusive( |
| 4791 | Type Ty, Operand *Addr, std::function<Variable *(Variable *)> Operation, |
| 4792 | CondARM32::Cond Cond) { |
| 4793 | |
| 4794 | auto *Retry = Context.insert<InstARM32Label>(this); |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 4795 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4796 | { // scoping for loop highlighting. |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 4797 | Variable *Success = makeReg(IceType_i32); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4798 | Variable *Tmp = (Ty == IceType_i64) ? makeI64RegPair() : makeReg(Ty); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4799 | auto *_0 = Ctx->getConstantZero(IceType_i32); |
| 4800 | |
| 4801 | Context.insert<InstFakeDef>(Tmp); |
| 4802 | Context.insert<InstFakeUse>(Tmp); |
| 4803 | Variable *AddrR = legalizeToReg(Addr); |
| 4804 | _ldrex(Tmp, formMemoryOperand(AddrR, Ty))->setDestRedefined(); |
| 4805 | auto *StoreValue = Operation(Tmp); |
| 4806 | assert(StoreValue->mustHaveReg()); |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 4807 | // strex requires Dest to be a register other than Value or Addr. This |
| 4808 | // restriction is cleanly represented by adding an "early" definition of |
| 4809 | // Dest (or a latter use of all the sources.) |
| 4810 | Context.insert<InstFakeDef>(Success); |
| 4811 | if (Cond != CondARM32::AL) { |
| 4812 | _mov_redefined(Success, legalize(_0, Legal_Reg | Legal_Flex), |
| 4813 | InstARM32::getOppositeCondition(Cond)); |
| 4814 | } |
| 4815 | _strex(Success, StoreValue, formMemoryOperand(AddrR, Ty), Cond) |
| 4816 | ->setDestRedefined(); |
| 4817 | _cmp(Success, _0); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4818 | } |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 4819 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4820 | _br(Retry, CondARM32::NE); |
| 4821 | } |
| 4822 | |
| 4823 | namespace { |
| 4824 | InstArithmetic *createArithInst(Cfg *Func, uint32_t Operation, Variable *Dest, |
| 4825 | Variable *Src0, Operand *Src1) { |
| 4826 | InstArithmetic::OpKind Oper; |
| 4827 | switch (Operation) { |
| 4828 | default: |
| 4829 | llvm::report_fatal_error("Unknown AtomicRMW operation"); |
| 4830 | case Intrinsics::AtomicExchange: |
| 4831 | llvm::report_fatal_error("Can't handle Atomic xchg operation"); |
| 4832 | case Intrinsics::AtomicAdd: |
| 4833 | Oper = InstArithmetic::Add; |
| 4834 | break; |
| 4835 | case Intrinsics::AtomicAnd: |
| 4836 | Oper = InstArithmetic::And; |
| 4837 | break; |
| 4838 | case Intrinsics::AtomicSub: |
| 4839 | Oper = InstArithmetic::Sub; |
| 4840 | break; |
| 4841 | case Intrinsics::AtomicOr: |
| 4842 | Oper = InstArithmetic::Or; |
| 4843 | break; |
| 4844 | case Intrinsics::AtomicXor: |
| 4845 | Oper = InstArithmetic::Xor; |
| 4846 | break; |
| 4847 | } |
| 4848 | return InstArithmetic::create(Func, Oper, Dest, Src0, Src1); |
| 4849 | } |
| 4850 | } // end of anonymous namespace |
| 4851 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4852 | void TargetARM32::lowerAtomicRMW(Variable *Dest, uint32_t Operation, |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4853 | Operand *Addr, Operand *Val) { |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4854 | // retry: |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4855 | // ldrex tmp, [addr] |
| 4856 | // mov contents, tmp |
| 4857 | // op result, contents, Val |
| 4858 | // strex success, result, [addr] |
| 4859 | // cmp success, 0 |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4860 | // jne retry |
| 4861 | // fake-use(addr, operand) @ prevents undesirable clobbering. |
| 4862 | // mov dest, contents |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4863 | auto DestTy = Dest->getType(); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4864 | |
| 4865 | if (DestTy == IceType_i64) { |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4866 | lowerInt64AtomicRMW(Dest, Operation, Addr, Val); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4867 | return; |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4868 | } |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4869 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4870 | Operand *ValRF = nullptr; |
| 4871 | if (llvm::isa<ConstantInteger32>(Val)) { |
| 4872 | ValRF = Val; |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4873 | } else { |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4874 | ValRF = legalizeToReg(Val); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4875 | } |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4876 | auto *ContentsR = makeReg(DestTy); |
| 4877 | auto *ResultR = makeReg(DestTy); |
| 4878 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4879 | _dmb(); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4880 | lowerLoadLinkedStoreExclusive( |
| 4881 | DestTy, Addr, |
| 4882 | [this, Operation, ResultR, ContentsR, ValRF](Variable *Tmp) { |
| 4883 | lowerAssign(InstAssign::create(Func, ContentsR, Tmp)); |
| 4884 | if (Operation == Intrinsics::AtomicExchange) { |
| 4885 | lowerAssign(InstAssign::create(Func, ResultR, ValRF)); |
| 4886 | } else { |
| 4887 | lowerArithmetic( |
| 4888 | createArithInst(Func, Operation, ResultR, ContentsR, ValRF)); |
| 4889 | } |
| 4890 | return ResultR; |
| 4891 | }); |
| 4892 | _dmb(); |
| 4893 | if (auto *ValR = llvm::dyn_cast<Variable>(ValRF)) { |
| 4894 | Context.insert<InstFakeUse>(ValR); |
| 4895 | } |
| 4896 | // Can't dce ContentsR. |
| 4897 | Context.insert<InstFakeUse>(ContentsR); |
| 4898 | lowerAssign(InstAssign::create(Func, Dest, ContentsR)); |
| 4899 | } |
| 4900 | |
| 4901 | void TargetARM32::lowerInt64AtomicRMW(Variable *Dest, uint32_t Operation, |
| 4902 | Operand *Addr, Operand *Val) { |
| 4903 | assert(Dest->getType() == IceType_i64); |
| 4904 | |
| 4905 | auto *ResultR = makeI64RegPair(); |
| 4906 | |
| 4907 | Context.insert<InstFakeDef>(ResultR); |
| 4908 | |
| 4909 | Operand *ValRF = nullptr; |
| 4910 | if (llvm::dyn_cast<ConstantInteger64>(Val)) { |
| 4911 | ValRF = Val; |
| 4912 | } else { |
| 4913 | auto *ValR64 = llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64)); |
| 4914 | ValR64->initHiLo(Func); |
| 4915 | ValR64->setMustNotHaveReg(); |
| 4916 | ValR64->getLo()->setMustHaveReg(); |
| 4917 | ValR64->getHi()->setMustHaveReg(); |
| 4918 | lowerAssign(InstAssign::create(Func, ValR64, Val)); |
| 4919 | ValRF = ValR64; |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4920 | } |
| 4921 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4922 | auto *ContentsR = llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64)); |
| 4923 | ContentsR->initHiLo(Func); |
| 4924 | ContentsR->setMustNotHaveReg(); |
| 4925 | ContentsR->getLo()->setMustHaveReg(); |
| 4926 | ContentsR->getHi()->setMustHaveReg(); |
| 4927 | |
| 4928 | _dmb(); |
| 4929 | lowerLoadLinkedStoreExclusive( |
| 4930 | IceType_i64, Addr, |
| 4931 | [this, Operation, ResultR, ContentsR, ValRF](Variable *Tmp) { |
| 4932 | lowerAssign(InstAssign::create(Func, ContentsR, Tmp)); |
| 4933 | Context.insert<InstFakeUse>(Tmp); |
| 4934 | if (Operation == Intrinsics::AtomicExchange) { |
| 4935 | lowerAssign(InstAssign::create(Func, ResultR, ValRF)); |
| 4936 | } else { |
| 4937 | lowerArithmetic( |
| 4938 | createArithInst(Func, Operation, ResultR, ContentsR, ValRF)); |
| 4939 | } |
| 4940 | Context.insert<InstFakeUse>(ResultR->getHi()); |
| 4941 | Context.insert<InstFakeDef>(ResultR, ResultR->getLo()) |
| 4942 | ->setDestRedefined(); |
| 4943 | return ResultR; |
| 4944 | }); |
| 4945 | _dmb(); |
| 4946 | if (auto *ValR64 = llvm::dyn_cast<Variable64On32>(ValRF)) { |
| 4947 | Context.insert<InstFakeUse>(ValR64->getLo()); |
| 4948 | Context.insert<InstFakeUse>(ValR64->getHi()); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4949 | } |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 4950 | lowerAssign(InstAssign::create(Func, Dest, ContentsR)); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4951 | } |
| 4952 | |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 4953 | void TargetARM32::postambleCtpop64(const InstCall *Instr) { |
| 4954 | Operand *Arg0 = Instr->getArg(0); |
| 4955 | if (isInt32Asserting32Or64(Arg0->getType())) { |
| 4956 | return; |
| 4957 | } |
| 4958 | // The popcount helpers always return 32-bit values, while the intrinsic's |
| 4959 | // signature matches some 64-bit platform's native instructions and expect to |
| 4960 | // fill a 64-bit reg. Thus, clear the upper bits of the dest just in case the |
| 4961 | // user doesn't do that in the IR or doesn't toss the bits via truncate. |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 4962 | auto *DestHi = llvm::cast<Variable>(hiOperand(Instr->getDest())); |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 4963 | Variable *T = makeReg(IceType_i32); |
| 4964 | Operand *_0 = |
| 4965 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
| 4966 | _mov(T, _0); |
| 4967 | _mov(DestHi, T); |
| 4968 | } |
| 4969 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4970 | void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) { |
| 4971 | Variable *Dest = Instr->getDest(); |
| 4972 | Type DestTy = (Dest != nullptr) ? Dest->getType() : IceType_void; |
| 4973 | Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID; |
| 4974 | switch (ID) { |
| 4975 | case Intrinsics::AtomicFence: |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4976 | case Intrinsics::AtomicFenceAll: |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4977 | assert(Dest == nullptr); |
| 4978 | _dmb(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 4979 | return; |
| 4980 | case Intrinsics::AtomicIsLockFree: { |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 4981 | Operand *ByteSize = Instr->getArg(0); |
| 4982 | auto *CI = llvm::dyn_cast<ConstantInteger32>(ByteSize); |
| 4983 | if (CI == nullptr) { |
| 4984 | // The PNaCl ABI requires the byte size to be a compile-time constant. |
| 4985 | Func->setError("AtomicIsLockFree byte size should be compile-time const"); |
| 4986 | return; |
| 4987 | } |
| 4988 | static constexpr int32_t NotLockFree = 0; |
| 4989 | static constexpr int32_t LockFree = 1; |
| 4990 | int32_t Result = NotLockFree; |
| 4991 | switch (CI->getValue()) { |
| 4992 | case 1: |
| 4993 | case 2: |
| 4994 | case 4: |
| 4995 | case 8: |
| 4996 | Result = LockFree; |
| 4997 | break; |
| 4998 | } |
| 4999 | _mov(Dest, legalizeToReg(Ctx->getConstantInt32(Result))); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5000 | return; |
| 5001 | } |
| 5002 | case Intrinsics::AtomicLoad: { |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5003 | assert(isScalarIntegerType(DestTy)); |
| 5004 | // We require the memory address to be naturally aligned. Given that is the |
| 5005 | // case, then normal loads are atomic. |
| 5006 | if (!Intrinsics::isMemoryOrderValid( |
| 5007 | ID, getConstantMemoryOrder(Instr->getArg(1)))) { |
| 5008 | Func->setError("Unexpected memory ordering for AtomicLoad"); |
| 5009 | return; |
| 5010 | } |
| 5011 | Variable *T; |
| 5012 | |
| 5013 | if (DestTy == IceType_i64) { |
| 5014 | // ldrex is the only arm instruction that is guaranteed to load a 64-bit |
| 5015 | // integer atomically. Everything else works with a regular ldr. |
| 5016 | T = makeI64RegPair(); |
| 5017 | _ldrex(T, formMemoryOperand(Instr->getArg(0), IceType_i64)); |
| 5018 | } else { |
| 5019 | T = makeReg(DestTy); |
| 5020 | _ldr(T, formMemoryOperand(Instr->getArg(0), DestTy)); |
| 5021 | } |
| 5022 | _dmb(); |
| 5023 | lowerAssign(InstAssign::create(Func, Dest, T)); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5024 | // Adding a fake-use T to ensure the atomic load is not removed if Dest is |
| 5025 | // unused. |
| 5026 | Context.insert<InstFakeUse>(T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5027 | return; |
| 5028 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5029 | case Intrinsics::AtomicStore: { |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5030 | // We require the memory address to be naturally aligned. Given that is the |
| 5031 | // case, then normal loads are atomic. |
| 5032 | if (!Intrinsics::isMemoryOrderValid( |
| 5033 | ID, getConstantMemoryOrder(Instr->getArg(2)))) { |
| 5034 | Func->setError("Unexpected memory ordering for AtomicStore"); |
| 5035 | return; |
| 5036 | } |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5037 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5038 | auto *Value = Instr->getArg(0); |
| 5039 | if (Value->getType() == IceType_i64) { |
| 5040 | auto *ValueR = makeI64RegPair(); |
| 5041 | Context.insert<InstFakeDef>(ValueR); |
| 5042 | lowerAssign(InstAssign::create(Func, ValueR, Value)); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5043 | _dmb(); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5044 | lowerLoadLinkedStoreExclusive( |
| 5045 | IceType_i64, Instr->getArg(1), [this, ValueR](Variable *Tmp) { |
| 5046 | // The following fake-use prevents the ldrex instruction from being |
| 5047 | // dead code eliminated. |
| 5048 | Context.insert<InstFakeUse>(llvm::cast<Variable>(loOperand(Tmp))); |
| 5049 | Context.insert<InstFakeUse>(llvm::cast<Variable>(hiOperand(Tmp))); |
| 5050 | Context.insert<InstFakeUse>(Tmp); |
| 5051 | return ValueR; |
| 5052 | }); |
| 5053 | Context.insert<InstFakeUse>(ValueR); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5054 | _dmb(); |
| 5055 | return; |
| 5056 | } |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5057 | |
| 5058 | auto *ValueR = legalizeToReg(Instr->getArg(0)); |
| 5059 | const auto ValueTy = ValueR->getType(); |
| 5060 | assert(isScalarIntegerType(ValueTy)); |
| 5061 | auto *Addr = legalizeToReg(Instr->getArg(1)); |
| 5062 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5063 | // non-64-bit stores are atomically as long as the address is aligned. This |
| 5064 | // is PNaCl, so addresses are aligned. |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5065 | _dmb(); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5066 | _str(ValueR, formMemoryOperand(Addr, ValueTy)); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5067 | _dmb(); |
| 5068 | return; |
| 5069 | } |
| 5070 | case Intrinsics::AtomicCmpxchg: { |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5071 | // retry: |
| 5072 | // ldrex tmp, [addr] |
| 5073 | // cmp tmp, expected |
| 5074 | // mov expected, tmp |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5075 | // strexeq success, new, [addr] |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5076 | // cmpeq success, #0 |
| 5077 | // bne retry |
| 5078 | // mov dest, expected |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5079 | assert(isScalarIntegerType(DestTy)); |
| 5080 | // We require the memory address to be naturally aligned. Given that is the |
| 5081 | // case, then normal loads are atomic. |
| 5082 | if (!Intrinsics::isMemoryOrderValid( |
| 5083 | ID, getConstantMemoryOrder(Instr->getArg(3)), |
| 5084 | getConstantMemoryOrder(Instr->getArg(4)))) { |
| 5085 | Func->setError("Unexpected memory ordering for AtomicCmpxchg"); |
| 5086 | return; |
| 5087 | } |
| 5088 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5089 | if (DestTy == IceType_i64) { |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5090 | Variable *LoadedValue = nullptr; |
| 5091 | |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5092 | auto *New = makeI64RegPair(); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5093 | Context.insert<InstFakeDef>(New); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5094 | lowerAssign(InstAssign::create(Func, New, Instr->getArg(2))); |
| 5095 | |
| 5096 | auto *Expected = makeI64RegPair(); |
| 5097 | Context.insert<InstFakeDef>(Expected); |
| 5098 | lowerAssign(InstAssign::create(Func, Expected, Instr->getArg(1))); |
| 5099 | |
| 5100 | _dmb(); |
| 5101 | lowerLoadLinkedStoreExclusive( |
| 5102 | DestTy, Instr->getArg(0), |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5103 | [this, Expected, New, Instr, DestTy, &LoadedValue](Variable *Tmp) { |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5104 | auto *ExpectedLoR = llvm::cast<Variable>(loOperand(Expected)); |
| 5105 | auto *ExpectedHiR = llvm::cast<Variable>(hiOperand(Expected)); |
| 5106 | auto *TmpLoR = llvm::cast<Variable>(loOperand(Tmp)); |
| 5107 | auto *TmpHiR = llvm::cast<Variable>(hiOperand(Tmp)); |
| 5108 | _cmp(TmpLoR, ExpectedLoR); |
| 5109 | _cmp(TmpHiR, ExpectedHiR, CondARM32::EQ); |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5110 | LoadedValue = Tmp; |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5111 | return New; |
| 5112 | }, |
| 5113 | CondARM32::EQ); |
| 5114 | _dmb(); |
| 5115 | |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5116 | Context.insert<InstFakeUse>(LoadedValue); |
| 5117 | lowerAssign(InstAssign::create(Func, Dest, LoadedValue)); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5118 | // The fake-use Expected prevents the assignments to Expected (above) |
| 5119 | // from being removed if Dest is not used. |
| 5120 | Context.insert<InstFakeUse>(Expected); |
| 5121 | // New needs to be alive here, or its live range will end in the |
| 5122 | // strex instruction. |
| 5123 | Context.insert<InstFakeUse>(New); |
| 5124 | return; |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5125 | } |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5126 | |
| 5127 | auto *New = legalizeToReg(Instr->getArg(2)); |
| 5128 | auto *Expected = legalizeToReg(Instr->getArg(1)); |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5129 | Variable *LoadedValue = nullptr; |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5130 | |
| 5131 | _dmb(); |
| 5132 | lowerLoadLinkedStoreExclusive( |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5133 | DestTy, Instr->getArg(0), |
| 5134 | [this, Expected, New, Instr, DestTy, &LoadedValue](Variable *Tmp) { |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5135 | lowerIcmpCond(InstIcmp::Eq, Tmp, Expected); |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5136 | LoadedValue = Tmp; |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5137 | return New; |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5138 | }, |
| 5139 | CondARM32::EQ); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5140 | _dmb(); |
| 5141 | |
John Porto | 324334e | 2016-03-08 11:00:53 -0800 | [diff] [blame] | 5142 | lowerAssign(InstAssign::create(Func, Dest, LoadedValue)); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5143 | Context.insert<InstFakeUse>(Expected); |
John Porto | 4b6e4b4 | 2016-02-17 05:00:59 -0800 | [diff] [blame] | 5144 | Context.insert<InstFakeUse>(New); |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5145 | return; |
| 5146 | } |
| 5147 | case Intrinsics::AtomicRMW: { |
| 5148 | if (!Intrinsics::isMemoryOrderValid( |
| 5149 | ID, getConstantMemoryOrder(Instr->getArg(3)))) { |
| 5150 | Func->setError("Unexpected memory ordering for AtomicRMW"); |
| 5151 | return; |
| 5152 | } |
| 5153 | lowerAtomicRMW( |
| 5154 | Dest, static_cast<uint32_t>( |
| 5155 | llvm::cast<ConstantInteger32>(Instr->getArg(0))->getValue()), |
| 5156 | Instr->getArg(1), Instr->getArg(2)); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5157 | return; |
| 5158 | } |
| 5159 | case Intrinsics::Bswap: { |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5160 | Operand *Val = Instr->getArg(0); |
| 5161 | Type Ty = Val->getType(); |
| 5162 | if (Ty == IceType_i64) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 5163 | Val = legalizeUndef(Val); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5164 | Variable *Val_Lo = legalizeToReg(loOperand(Val)); |
| 5165 | Variable *Val_Hi = legalizeToReg(hiOperand(Val)); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5166 | Variable *T_Lo = makeReg(IceType_i32); |
| 5167 | Variable *T_Hi = makeReg(IceType_i32); |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 5168 | auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 5169 | auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5170 | _rev(T_Lo, Val_Lo); |
| 5171 | _rev(T_Hi, Val_Hi); |
| 5172 | _mov(DestLo, T_Hi); |
| 5173 | _mov(DestHi, T_Lo); |
| 5174 | } else { |
| 5175 | assert(Ty == IceType_i32 || Ty == IceType_i16); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5176 | Variable *ValR = legalizeToReg(Val); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5177 | Variable *T = makeReg(Ty); |
| 5178 | _rev(T, ValR); |
| 5179 | if (Val->getType() == IceType_i16) { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 5180 | Operand *_16 = shAmtImm(16); |
| 5181 | _lsr(T, T, _16); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5182 | } |
| 5183 | _mov(Dest, T); |
| 5184 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5185 | return; |
| 5186 | } |
| 5187 | case Intrinsics::Ctpop: { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 5188 | llvm::report_fatal_error("Ctpop should have been prelowered."); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5189 | } |
| 5190 | case Intrinsics::Ctlz: { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 5191 | // The "is zero undef" parameter is ignored and we always return a |
| 5192 | // well-defined value. |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5193 | Operand *Val = Instr->getArg(0); |
| 5194 | Variable *ValLoR; |
| 5195 | Variable *ValHiR = nullptr; |
| 5196 | if (Val->getType() == IceType_i64) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 5197 | Val = legalizeUndef(Val); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5198 | ValLoR = legalizeToReg(loOperand(Val)); |
| 5199 | ValHiR = legalizeToReg(hiOperand(Val)); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5200 | } else { |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5201 | ValLoR = legalizeToReg(Val); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5202 | } |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5203 | lowerCLZ(Dest, ValLoR, ValHiR); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5204 | return; |
| 5205 | } |
| 5206 | case Intrinsics::Cttz: { |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5207 | // Essentially like Clz, but reverse the bits first. |
| 5208 | Operand *Val = Instr->getArg(0); |
| 5209 | Variable *ValLoR; |
| 5210 | Variable *ValHiR = nullptr; |
| 5211 | if (Val->getType() == IceType_i64) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 5212 | Val = legalizeUndef(Val); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5213 | ValLoR = legalizeToReg(loOperand(Val)); |
| 5214 | ValHiR = legalizeToReg(hiOperand(Val)); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5215 | Variable *TLo = makeReg(IceType_i32); |
| 5216 | Variable *THi = makeReg(IceType_i32); |
| 5217 | _rbit(TLo, ValLoR); |
| 5218 | _rbit(THi, ValHiR); |
| 5219 | ValLoR = THi; |
| 5220 | ValHiR = TLo; |
| 5221 | } else { |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5222 | ValLoR = legalizeToReg(Val); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5223 | Variable *T = makeReg(IceType_i32); |
| 5224 | _rbit(T, ValLoR); |
| 5225 | ValLoR = T; |
| 5226 | } |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 5227 | lowerCLZ(Dest, ValLoR, ValHiR); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5228 | return; |
| 5229 | } |
| 5230 | case Intrinsics::Fabs: { |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 5231 | Type DestTy = Dest->getType(); |
| 5232 | Variable *T = makeReg(DestTy); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 5233 | _vabs(T, legalizeToReg(Instr->getArg(0))); |
| 5234 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5235 | return; |
| 5236 | } |
| 5237 | case Intrinsics::Longjmp: { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 5238 | llvm::report_fatal_error("longjmp should have been prelowered."); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5239 | } |
| 5240 | case Intrinsics::Memcpy: { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 5241 | llvm::report_fatal_error("memcpy should have been prelowered."); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5242 | } |
| 5243 | case Intrinsics::Memmove: { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 5244 | llvm::report_fatal_error("memmove should have been prelowered."); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5245 | } |
| 5246 | case Intrinsics::Memset: { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 5247 | llvm::report_fatal_error("memmove should have been prelowered."); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5248 | } |
| 5249 | case Intrinsics::NaClReadTP: { |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 5250 | if (SandboxingType != ST_NaCl) { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 5251 | llvm::report_fatal_error("nacl-read-tp should have been prelowered."); |
| 5252 | } |
| 5253 | Variable *TP = legalizeToReg(OperandARM32Mem::create( |
| 5254 | Func, getPointerType(), getPhysicalRegister(RegARM32::Reg_r9), |
| 5255 | llvm::cast<ConstantInteger32>(Ctx->getConstantZero(IceType_i32)))); |
| 5256 | _mov(Dest, TP); |
| 5257 | return; |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5258 | } |
| 5259 | case Intrinsics::Setjmp: { |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 5260 | llvm::report_fatal_error("setjmp should have been prelowered."); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5261 | } |
| 5262 | case Intrinsics::Sqrt: { |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 5263 | Variable *Src = legalizeToReg(Instr->getArg(0)); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 5264 | Variable *T = makeReg(Dest->getType()); |
| 5265 | _vsqrt(T, Src); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 5266 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5267 | return; |
| 5268 | } |
| 5269 | case Intrinsics::Stacksave: { |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5270 | Variable *SP = getPhysicalRegister(RegARM32::Reg_sp); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5271 | _mov(Dest, SP); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5272 | return; |
| 5273 | } |
| 5274 | case Intrinsics::Stackrestore: { |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 5275 | Variable *Val = legalizeToReg(Instr->getArg(0)); |
| 5276 | Sandboxer(this).reset_sp(Val); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5277 | return; |
| 5278 | } |
| 5279 | case Intrinsics::Trap: |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5280 | _trap(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5281 | return; |
| 5282 | case Intrinsics::UnknownIntrinsic: |
| 5283 | Func->setError("Should not be lowering UnknownIntrinsic"); |
| 5284 | return; |
| 5285 | } |
| 5286 | return; |
| 5287 | } |
| 5288 | |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5289 | void TargetARM32::lowerCLZ(Variable *Dest, Variable *ValLoR, Variable *ValHiR) { |
| 5290 | Type Ty = Dest->getType(); |
| 5291 | assert(Ty == IceType_i32 || Ty == IceType_i64); |
| 5292 | Variable *T = makeReg(IceType_i32); |
| 5293 | _clz(T, ValLoR); |
| 5294 | if (Ty == IceType_i64) { |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 5295 | auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 5296 | auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5297 | Operand *Zero = |
| 5298 | legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex); |
| 5299 | Operand *ThirtyTwo = |
| 5300 | legalize(Ctx->getConstantInt32(32), Legal_Reg | Legal_Flex); |
| 5301 | _cmp(ValHiR, Zero); |
| 5302 | Variable *T2 = makeReg(IceType_i32); |
| 5303 | _add(T2, T, ThirtyTwo); |
| 5304 | _clz(T2, ValHiR, CondARM32::NE); |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 5305 | // T2 is actually a source as well when the predicate is not AL (since it |
Jim Stichnoth | 230d410 | 2015-09-25 17:40:32 -0700 | [diff] [blame] | 5306 | // may leave T2 alone). We use _set_dest_redefined to prolong the liveness |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 5307 | // of T2 as if it was used as a source. |
Jim Stichnoth | 230d410 | 2015-09-25 17:40:32 -0700 | [diff] [blame] | 5308 | _set_dest_redefined(); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5309 | _mov(DestLo, T2); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 5310 | Variable *T3 = makeReg(Zero->getType()); |
Jan Voung | 28068ad | 2015-07-31 12:58:46 -0700 | [diff] [blame] | 5311 | _mov(T3, Zero); |
| 5312 | _mov(DestHi, T3); |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5313 | return; |
| 5314 | } |
| 5315 | _mov(Dest, T); |
| 5316 | return; |
| 5317 | } |
| 5318 | |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 5319 | void TargetARM32::lowerLoad(const InstLoad *Load) { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 5320 | // A Load instruction can be treated the same as an Assign instruction, after |
| 5321 | // the source operand is transformed into an OperandARM32Mem operand. |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 5322 | Type Ty = Load->getDest()->getType(); |
| 5323 | Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); |
| 5324 | Variable *DestLoad = Load->getDest(); |
| 5325 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 5326 | // TODO(jvoung): handled folding opportunities. Sign and zero extension can |
| 5327 | // be folded into a load. |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 5328 | auto *Assign = InstAssign::create(Func, DestLoad, Src0); |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 5329 | lowerAssign(Assign); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5330 | } |
| 5331 | |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5332 | namespace { |
| 5333 | void dumpAddressOpt(const Cfg *Func, const Variable *Base, int32_t Offset, |
| 5334 | const Variable *OffsetReg, int16_t OffsetRegShAmt, |
| 5335 | const Inst *Reason) { |
| 5336 | if (!BuildDefs::dump()) |
| 5337 | return; |
| 5338 | if (!Func->isVerbose(IceV_AddrOpt)) |
| 5339 | return; |
| 5340 | OstreamLocker _(Func->getContext()); |
| 5341 | Ostream &Str = Func->getContext()->getStrDump(); |
| 5342 | Str << "Instruction: "; |
| 5343 | Reason->dumpDecorated(Func); |
| 5344 | Str << " results in Base="; |
| 5345 | if (Base) |
| 5346 | Base->dump(Func); |
| 5347 | else |
| 5348 | Str << "<null>"; |
| 5349 | Str << ", OffsetReg="; |
| 5350 | if (OffsetReg) |
| 5351 | OffsetReg->dump(Func); |
| 5352 | else |
| 5353 | Str << "<null>"; |
| 5354 | Str << ", Shift=" << OffsetRegShAmt << ", Offset=" << Offset << "\n"; |
| 5355 | } |
| 5356 | |
| 5357 | bool matchAssign(const VariablesMetadata *VMetadata, Variable **Var, |
| 5358 | int32_t *Offset, const Inst **Reason) { |
| 5359 | // Var originates from Var=SrcVar ==> set Var:=SrcVar |
| 5360 | if (*Var == nullptr) |
| 5361 | return false; |
| 5362 | const Inst *VarAssign = VMetadata->getSingleDefinition(*Var); |
| 5363 | if (!VarAssign) |
| 5364 | return false; |
| 5365 | assert(!VMetadata->isMultiDef(*Var)); |
| 5366 | if (!llvm::isa<InstAssign>(VarAssign)) |
| 5367 | return false; |
| 5368 | |
| 5369 | Operand *SrcOp = VarAssign->getSrc(0); |
John Porto | 6f53471 | 2016-01-20 10:49:38 -0800 | [diff] [blame] | 5370 | bool Optimized = false; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5371 | if (auto *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { |
| 5372 | if (!VMetadata->isMultiDef(SrcVar) || |
| 5373 | // TODO: ensure SrcVar stays single-BB |
| 5374 | false) { |
John Porto | 6f53471 | 2016-01-20 10:49:38 -0800 | [diff] [blame] | 5375 | Optimized = true; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5376 | *Var = SrcVar; |
| 5377 | } else if (auto *Const = llvm::dyn_cast<ConstantInteger32>(SrcOp)) { |
| 5378 | int32_t MoreOffset = Const->getValue(); |
| 5379 | int32_t NewOffset = MoreOffset + *Offset; |
| 5380 | if (Utils::WouldOverflowAdd(*Offset, MoreOffset)) |
| 5381 | return false; |
| 5382 | *Var = nullptr; |
| 5383 | *Offset += NewOffset; |
John Porto | 6f53471 | 2016-01-20 10:49:38 -0800 | [diff] [blame] | 5384 | Optimized = true; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5385 | } |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5386 | } |
| 5387 | |
John Porto | 6f53471 | 2016-01-20 10:49:38 -0800 | [diff] [blame] | 5388 | if (Optimized) { |
| 5389 | *Reason = VarAssign; |
| 5390 | } |
| 5391 | |
| 5392 | return Optimized; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5393 | } |
| 5394 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5395 | bool isAddOrSub(const Inst *Instr, InstArithmetic::OpKind *Kind) { |
| 5396 | if (const auto *Arith = llvm::dyn_cast<InstArithmetic>(Instr)) { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5397 | switch (Arith->getOp()) { |
| 5398 | default: |
| 5399 | return false; |
| 5400 | case InstArithmetic::Add: |
| 5401 | case InstArithmetic::Sub: |
| 5402 | *Kind = Arith->getOp(); |
| 5403 | return true; |
| 5404 | } |
| 5405 | } |
| 5406 | return false; |
| 5407 | } |
| 5408 | |
| 5409 | bool matchCombinedBaseIndex(const VariablesMetadata *VMetadata, Variable **Base, |
| 5410 | Variable **OffsetReg, int32_t OffsetRegShamt, |
| 5411 | const Inst **Reason) { |
| 5412 | // OffsetReg==nullptr && Base is Base=Var1+Var2 ==> |
| 5413 | // set Base=Var1, OffsetReg=Var2, Shift=0 |
| 5414 | if (*Base == nullptr) |
| 5415 | return false; |
| 5416 | if (*OffsetReg != nullptr) |
| 5417 | return false; |
| 5418 | (void)OffsetRegShamt; |
| 5419 | assert(OffsetRegShamt == 0); |
| 5420 | const Inst *BaseInst = VMetadata->getSingleDefinition(*Base); |
| 5421 | if (BaseInst == nullptr) |
| 5422 | return false; |
| 5423 | assert(!VMetadata->isMultiDef(*Base)); |
| 5424 | if (BaseInst->getSrcSize() < 2) |
| 5425 | return false; |
| 5426 | auto *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0)); |
| 5427 | if (!Var1) |
| 5428 | return false; |
| 5429 | if (VMetadata->isMultiDef(Var1)) |
| 5430 | return false; |
| 5431 | auto *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1)); |
| 5432 | if (!Var2) |
| 5433 | return false; |
| 5434 | if (VMetadata->isMultiDef(Var2)) |
| 5435 | return false; |
| 5436 | InstArithmetic::OpKind _; |
| 5437 | if (!isAddOrSub(BaseInst, &_) || |
| 5438 | // TODO: ensure Var1 and Var2 stay single-BB |
| 5439 | false) |
| 5440 | return false; |
| 5441 | *Base = Var1; |
| 5442 | *OffsetReg = Var2; |
| 5443 | // OffsetRegShamt is already 0. |
| 5444 | *Reason = BaseInst; |
| 5445 | return true; |
| 5446 | } |
| 5447 | |
| 5448 | bool matchShiftedOffsetReg(const VariablesMetadata *VMetadata, |
| 5449 | Variable **OffsetReg, OperandARM32::ShiftKind *Kind, |
| 5450 | int32_t *OffsetRegShamt, const Inst **Reason) { |
| 5451 | // OffsetReg is OffsetReg=Var*Const && log2(Const)+Shift<=32 ==> |
| 5452 | // OffsetReg=Var, Shift+=log2(Const) |
| 5453 | // OffsetReg is OffsetReg=Var<<Const && Const+Shift<=32 ==> |
| 5454 | // OffsetReg=Var, Shift+=Const |
| 5455 | // OffsetReg is OffsetReg=Var>>Const && Const-Shift>=-32 ==> |
| 5456 | // OffsetReg=Var, Shift-=Const |
| 5457 | OperandARM32::ShiftKind NewShiftKind = OperandARM32::kNoShift; |
| 5458 | if (*OffsetReg == nullptr) |
| 5459 | return false; |
| 5460 | auto *IndexInst = VMetadata->getSingleDefinition(*OffsetReg); |
| 5461 | if (IndexInst == nullptr) |
| 5462 | return false; |
| 5463 | assert(!VMetadata->isMultiDef(*OffsetReg)); |
| 5464 | if (IndexInst->getSrcSize() < 2) |
| 5465 | return false; |
| 5466 | auto *ArithInst = llvm::dyn_cast<InstArithmetic>(IndexInst); |
| 5467 | if (ArithInst == nullptr) |
| 5468 | return false; |
| 5469 | auto *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0)); |
| 5470 | if (Var == nullptr) |
| 5471 | return false; |
| 5472 | auto *Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1)); |
| 5473 | if (Const == nullptr) { |
| 5474 | assert(!llvm::isa<ConstantInteger32>(ArithInst->getSrc(0))); |
| 5475 | return false; |
| 5476 | } |
| 5477 | if (VMetadata->isMultiDef(Var) || Const->getType() != IceType_i32) |
| 5478 | return false; |
| 5479 | |
| 5480 | uint32_t NewShamt = -1; |
| 5481 | switch (ArithInst->getOp()) { |
| 5482 | default: |
| 5483 | return false; |
| 5484 | case InstArithmetic::Shl: { |
| 5485 | NewShiftKind = OperandARM32::LSL; |
| 5486 | NewShamt = Const->getValue(); |
| 5487 | if (NewShamt > 31) |
| 5488 | return false; |
| 5489 | } break; |
| 5490 | case InstArithmetic::Lshr: { |
| 5491 | NewShiftKind = OperandARM32::LSR; |
| 5492 | NewShamt = Const->getValue(); |
| 5493 | if (NewShamt > 31) |
| 5494 | return false; |
| 5495 | } break; |
| 5496 | case InstArithmetic::Ashr: { |
| 5497 | NewShiftKind = OperandARM32::ASR; |
| 5498 | NewShamt = Const->getValue(); |
| 5499 | if (NewShamt > 31) |
| 5500 | return false; |
| 5501 | } break; |
| 5502 | case InstArithmetic::Udiv: |
| 5503 | case InstArithmetic::Mul: { |
| 5504 | const uint32_t UnsignedConst = Const->getValue(); |
| 5505 | NewShamt = llvm::findFirstSet(UnsignedConst); |
| 5506 | if (NewShamt != llvm::findLastSet(UnsignedConst)) { |
| 5507 | // First bit set is not the same as the last bit set, so Const is not |
| 5508 | // a power of 2. |
| 5509 | return false; |
| 5510 | } |
| 5511 | NewShiftKind = ArithInst->getOp() == InstArithmetic::Udiv |
| 5512 | ? OperandARM32::LSR |
| 5513 | : OperandARM32::LSL; |
| 5514 | } break; |
| 5515 | } |
| 5516 | // Allowed "transitions": |
| 5517 | // kNoShift -> * iff NewShamt < 31 |
| 5518 | // LSL -> LSL iff NewShamt + OffsetRegShamt < 31 |
| 5519 | // LSR -> LSR iff NewShamt + OffsetRegShamt < 31 |
| 5520 | // ASR -> ASR iff NewShamt + OffsetRegShamt < 31 |
| 5521 | if (*Kind != OperandARM32::kNoShift && *Kind != NewShiftKind) { |
| 5522 | return false; |
| 5523 | } |
| 5524 | const int32_t NewOffsetRegShamt = *OffsetRegShamt + NewShamt; |
| 5525 | if (NewOffsetRegShamt > 31) |
| 5526 | return false; |
| 5527 | *OffsetReg = Var; |
| 5528 | *OffsetRegShamt = NewOffsetRegShamt; |
| 5529 | *Kind = NewShiftKind; |
| 5530 | *Reason = IndexInst; |
| 5531 | return true; |
| 5532 | } |
| 5533 | |
| 5534 | bool matchOffsetBase(const VariablesMetadata *VMetadata, Variable **Base, |
| 5535 | int32_t *Offset, const Inst **Reason) { |
| 5536 | // Base is Base=Var+Const || Base is Base=Const+Var ==> |
| 5537 | // set Base=Var, Offset+=Const |
| 5538 | // Base is Base=Var-Const ==> |
| 5539 | // set Base=Var, Offset-=Const |
| 5540 | if (*Base == nullptr) |
| 5541 | return false; |
| 5542 | const Inst *BaseInst = VMetadata->getSingleDefinition(*Base); |
| 5543 | if (BaseInst == nullptr) { |
| 5544 | return false; |
| 5545 | } |
| 5546 | assert(!VMetadata->isMultiDef(*Base)); |
| 5547 | |
| 5548 | auto *ArithInst = llvm::dyn_cast<const InstArithmetic>(BaseInst); |
| 5549 | if (ArithInst == nullptr) |
| 5550 | return false; |
| 5551 | InstArithmetic::OpKind Kind; |
| 5552 | if (!isAddOrSub(ArithInst, &Kind)) |
| 5553 | return false; |
| 5554 | bool IsAdd = Kind == InstArithmetic::Add; |
| 5555 | Operand *Src0 = ArithInst->getSrc(0); |
| 5556 | Operand *Src1 = ArithInst->getSrc(1); |
| 5557 | auto *Var0 = llvm::dyn_cast<Variable>(Src0); |
| 5558 | auto *Var1 = llvm::dyn_cast<Variable>(Src1); |
| 5559 | auto *Const0 = llvm::dyn_cast<ConstantInteger32>(Src0); |
| 5560 | auto *Const1 = llvm::dyn_cast<ConstantInteger32>(Src1); |
| 5561 | Variable *NewBase = nullptr; |
| 5562 | int32_t NewOffset = *Offset; |
| 5563 | |
| 5564 | if (Var0 == nullptr && Const0 == nullptr) { |
| 5565 | assert(llvm::isa<ConstantRelocatable>(Src0)); |
| 5566 | return false; |
| 5567 | } |
| 5568 | |
| 5569 | if (Var1 == nullptr && Const1 == nullptr) { |
| 5570 | assert(llvm::isa<ConstantRelocatable>(Src1)); |
| 5571 | return false; |
| 5572 | } |
| 5573 | |
| 5574 | if (Var0 && Var1) |
| 5575 | // TODO(jpp): merge base/index splitting into here. |
| 5576 | return false; |
| 5577 | if (!IsAdd && Var1) |
| 5578 | return false; |
| 5579 | if (Var0) |
| 5580 | NewBase = Var0; |
| 5581 | else if (Var1) |
| 5582 | NewBase = Var1; |
| 5583 | // Compute the updated constant offset. |
| 5584 | if (Const0) { |
| 5585 | int32_t MoreOffset = IsAdd ? Const0->getValue() : -Const0->getValue(); |
| 5586 | if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
| 5587 | return false; |
| 5588 | NewOffset += MoreOffset; |
| 5589 | } |
| 5590 | if (Const1) { |
| 5591 | int32_t MoreOffset = IsAdd ? Const1->getValue() : -Const1->getValue(); |
| 5592 | if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
| 5593 | return false; |
| 5594 | NewOffset += MoreOffset; |
| 5595 | } |
| 5596 | |
| 5597 | // Update the computed address parameters once we are sure optimization |
| 5598 | // is valid. |
| 5599 | *Base = NewBase; |
| 5600 | *Offset = NewOffset; |
| 5601 | *Reason = BaseInst; |
| 5602 | return true; |
| 5603 | } |
| 5604 | } // end of anonymous namespace |
| 5605 | |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5606 | OperandARM32Mem *TargetARM32::formAddressingMode(Type Ty, Cfg *Func, |
| 5607 | const Inst *LdSt, |
| 5608 | Operand *Base) { |
| 5609 | assert(Base != nullptr); |
| 5610 | int32_t OffsetImm = 0; |
| 5611 | Variable *OffsetReg = nullptr; |
| 5612 | int32_t OffsetRegShamt = 0; |
| 5613 | OperandARM32::ShiftKind ShiftKind = OperandARM32::kNoShift; |
| 5614 | |
| 5615 | Func->resetCurrentNode(); |
| 5616 | if (Func->isVerbose(IceV_AddrOpt)) { |
| 5617 | OstreamLocker _(Func->getContext()); |
| 5618 | Ostream &Str = Func->getContext()->getStrDump(); |
| 5619 | Str << "\nAddress mode formation:\t"; |
| 5620 | LdSt->dumpDecorated(Func); |
| 5621 | } |
| 5622 | |
| 5623 | if (isVectorType(Ty)) |
| 5624 | // vector loads and stores do not allow offsets, and only support the |
| 5625 | // "[reg]" addressing mode (the other supported modes are write back.) |
| 5626 | return nullptr; |
| 5627 | |
| 5628 | auto *BaseVar = llvm::dyn_cast<Variable>(Base); |
| 5629 | if (BaseVar == nullptr) |
| 5630 | return nullptr; |
| 5631 | |
| 5632 | (void)MemTraitsSize; |
| 5633 | assert(Ty < MemTraitsSize); |
| 5634 | auto *TypeTraits = &MemTraits[Ty]; |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 5635 | const bool CanHaveIndex = !NeedSandboxing && TypeTraits->CanHaveIndex; |
| 5636 | const bool CanHaveShiftedIndex = |
| 5637 | !NeedSandboxing && TypeTraits->CanHaveShiftedIndex; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5638 | const bool CanHaveImm = TypeTraits->CanHaveImm; |
| 5639 | const int32_t ValidImmMask = TypeTraits->ValidImmMask; |
| 5640 | (void)ValidImmMask; |
| 5641 | assert(!CanHaveImm || ValidImmMask >= 0); |
| 5642 | |
| 5643 | const VariablesMetadata *VMetadata = Func->getVMetadata(); |
| 5644 | const Inst *Reason = nullptr; |
| 5645 | |
| 5646 | do { |
| 5647 | if (Reason != nullptr) { |
| 5648 | dumpAddressOpt(Func, BaseVar, OffsetImm, OffsetReg, OffsetRegShamt, |
| 5649 | Reason); |
| 5650 | Reason = nullptr; |
| 5651 | } |
| 5652 | |
| 5653 | if (matchAssign(VMetadata, &BaseVar, &OffsetImm, &Reason)) { |
| 5654 | continue; |
| 5655 | } |
| 5656 | |
| 5657 | if (CanHaveIndex && |
| 5658 | matchAssign(VMetadata, &OffsetReg, &OffsetImm, &Reason)) { |
| 5659 | continue; |
| 5660 | } |
| 5661 | |
| 5662 | if (CanHaveIndex && matchCombinedBaseIndex(VMetadata, &BaseVar, &OffsetReg, |
| 5663 | OffsetRegShamt, &Reason)) { |
| 5664 | continue; |
| 5665 | } |
| 5666 | |
| 5667 | if (CanHaveShiftedIndex) { |
| 5668 | if (matchShiftedOffsetReg(VMetadata, &OffsetReg, &ShiftKind, |
| 5669 | &OffsetRegShamt, &Reason)) { |
| 5670 | continue; |
| 5671 | } |
| 5672 | |
| 5673 | if ((OffsetRegShamt == 0) && |
| 5674 | matchShiftedOffsetReg(VMetadata, &BaseVar, &ShiftKind, |
| 5675 | &OffsetRegShamt, &Reason)) { |
| 5676 | std::swap(BaseVar, OffsetReg); |
| 5677 | continue; |
| 5678 | } |
| 5679 | } |
| 5680 | |
| 5681 | if (matchOffsetBase(VMetadata, &BaseVar, &OffsetImm, &Reason)) { |
| 5682 | continue; |
| 5683 | } |
| 5684 | } while (Reason); |
| 5685 | |
| 5686 | if (BaseVar == nullptr) { |
| 5687 | // [OffsetReg{, LSL Shamt}{, #OffsetImm}] is not legal in ARM, so we have to |
| 5688 | // legalize the addressing mode to [BaseReg, OffsetReg{, LSL Shamt}]. |
| 5689 | // Instead of a zeroed BaseReg, we initialize it with OffsetImm: |
| 5690 | // |
| 5691 | // [OffsetReg{, LSL Shamt}{, #OffsetImm}] -> |
| 5692 | // mov BaseReg, #OffsetImm |
| 5693 | // use of [BaseReg, OffsetReg{, LSL Shamt}] |
| 5694 | // |
| 5695 | const Type PointerType = getPointerType(); |
| 5696 | BaseVar = makeReg(PointerType); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5697 | Context.insert<InstAssign>(BaseVar, Ctx->getConstantInt32(OffsetImm)); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5698 | OffsetImm = 0; |
| 5699 | } else if (OffsetImm != 0) { |
| 5700 | // ARM Ldr/Str instructions have limited range immediates. The formation |
| 5701 | // loop above materialized an Immediate carelessly, so we ensure the |
| 5702 | // generated offset is sane. |
| 5703 | const int32_t PositiveOffset = OffsetImm > 0 ? OffsetImm : -OffsetImm; |
| 5704 | const InstArithmetic::OpKind Op = |
| 5705 | OffsetImm > 0 ? InstArithmetic::Add : InstArithmetic::Sub; |
| 5706 | |
| 5707 | if (!CanHaveImm || !isLegalMemOffset(Ty, OffsetImm) || |
| 5708 | OffsetReg != nullptr) { |
| 5709 | if (OffsetReg == nullptr) { |
| 5710 | // We formed a [Base, #const] addressing mode which is not encodable in |
| 5711 | // ARM. There is little point in forming an address mode now if we don't |
| 5712 | // have an offset. Effectively, we would end up with something like |
| 5713 | // |
| 5714 | // [Base, #const] -> add T, Base, #const |
| 5715 | // use of [T] |
| 5716 | // |
| 5717 | // Which is exactly what we already have. So we just bite the bullet |
| 5718 | // here and don't form any address mode. |
| 5719 | return nullptr; |
| 5720 | } |
| 5721 | // We formed [Base, Offset {, LSL Amnt}, #const]. Oops. Legalize it to |
| 5722 | // |
| 5723 | // [Base, Offset, {LSL amount}, #const] -> |
| 5724 | // add T, Base, #const |
| 5725 | // use of [T, Offset {, LSL amount}] |
| 5726 | const Type PointerType = getPointerType(); |
| 5727 | Variable *T = makeReg(PointerType); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5728 | Context.insert<InstArithmetic>(Op, T, BaseVar, |
| 5729 | Ctx->getConstantInt32(PositiveOffset)); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5730 | BaseVar = T; |
| 5731 | OffsetImm = 0; |
| 5732 | } |
| 5733 | } |
| 5734 | |
| 5735 | assert(BaseVar != nullptr); |
| 5736 | assert(OffsetImm == 0 || OffsetReg == nullptr); |
| 5737 | assert(OffsetReg == nullptr || CanHaveIndex); |
| 5738 | assert(OffsetImm < 0 ? (ValidImmMask & -OffsetImm) == -OffsetImm |
| 5739 | : (ValidImmMask & OffsetImm) == OffsetImm); |
| 5740 | |
| 5741 | if (OffsetReg != nullptr) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 5742 | Variable *OffsetR = makeReg(getPointerType()); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5743 | Context.insert<InstAssign>(OffsetR, OffsetReg); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 5744 | return OperandARM32Mem::create(Func, Ty, BaseVar, OffsetR, ShiftKind, |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5745 | OffsetRegShamt); |
| 5746 | } |
| 5747 | |
| 5748 | return OperandARM32Mem::create( |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 5749 | Func, Ty, BaseVar, |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5750 | llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(OffsetImm))); |
| 5751 | } |
| 5752 | |
| 5753 | void TargetARM32::doAddressOptLoad() { |
Jim Stichnoth | f5fdd23 | 2016-05-09 12:24:36 -0700 | [diff] [blame] | 5754 | Inst *Instr = iteratorToInst(Context.getCur()); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5755 | assert(llvm::isa<InstLoad>(Instr)); |
| 5756 | Variable *Dest = Instr->getDest(); |
| 5757 | Operand *Addr = Instr->getSrc(0); |
| 5758 | if (OperandARM32Mem *Mem = |
| 5759 | formAddressingMode(Dest->getType(), Func, Instr, Addr)) { |
| 5760 | Instr->setDeleted(); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5761 | Context.insert<InstLoad>(Dest, Mem); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5762 | } |
| 5763 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5764 | |
Qining Lu | aee5fa8 | 2015-08-20 14:59:03 -0700 | [diff] [blame] | 5765 | void TargetARM32::randomlyInsertNop(float Probability, |
| 5766 | RandomNumberGenerator &RNG) { |
| 5767 | RandomNumberGeneratorWrapper RNGW(RNG); |
| 5768 | if (RNGW.getTrueWithProbability(Probability)) { |
Karl Schimpf | f084a57 | 2016-02-09 13:09:23 -0800 | [diff] [blame] | 5769 | _nop(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5770 | } |
| 5771 | } |
| 5772 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5773 | void TargetARM32::lowerPhi(const InstPhi * /*Instr*/) { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5774 | Func->setError("Phi found in regular instruction list"); |
| 5775 | } |
| 5776 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5777 | void TargetARM32::lowerRet(const InstRet *Instr) { |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 5778 | Variable *Reg = nullptr; |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5779 | if (Instr->hasRetValue()) { |
| 5780 | Operand *Src0 = Instr->getRetValue(); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 5781 | Type Ty = Src0->getType(); |
| 5782 | if (Ty == IceType_i64) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 5783 | Src0 = legalizeUndef(Src0); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5784 | Variable *R0 = legalizeToReg(loOperand(Src0), RegARM32::Reg_r0); |
| 5785 | Variable *R1 = legalizeToReg(hiOperand(Src0), RegARM32::Reg_r1); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 5786 | Reg = R0; |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5787 | Context.insert<InstFakeUse>(R1); |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 5788 | } else if (Ty == IceType_f32) { |
| 5789 | Variable *S0 = legalizeToReg(Src0, RegARM32::Reg_s0); |
| 5790 | Reg = S0; |
| 5791 | } else if (Ty == IceType_f64) { |
| 5792 | Variable *D0 = legalizeToReg(Src0, RegARM32::Reg_d0); |
| 5793 | Reg = D0; |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 5794 | } else if (isVectorType(Src0->getType())) { |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 5795 | Variable *Q0 = legalizeToReg(Src0, RegARM32::Reg_q0); |
| 5796 | Reg = Q0; |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 5797 | } else { |
| 5798 | Operand *Src0F = legalize(Src0, Legal_Reg | Legal_Flex); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 5799 | Reg = makeReg(Src0F->getType(), RegARM32::Reg_r0); |
| 5800 | _mov(Reg, Src0F, CondARM32::AL); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 5801 | } |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 5802 | } |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 5803 | // Add a ret instruction even if sandboxing is enabled, because addEpilog |
| 5804 | // explicitly looks for a ret instruction as a marker for where to insert the |
| 5805 | // frame removal instructions. addEpilog is responsible for restoring the |
| 5806 | // "lr" register as needed prior to this ret instruction. |
Jan Voung | b2d5084 | 2015-05-12 09:53:50 -0700 | [diff] [blame] | 5807 | _ret(getPhysicalRegister(RegARM32::Reg_lr), Reg); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 5808 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 5809 | // Add a fake use of sp to make sure sp stays alive for the entire function. |
| 5810 | // Otherwise post-call sp adjustments get dead-code eliminated. |
| 5811 | // TODO: Are there more places where the fake use should be inserted? E.g. |
| 5812 | // "void f(int n){while(1) g(n);}" may not have a ret instruction. |
Jan Voung | f645d85 | 2015-07-09 10:35:09 -0700 | [diff] [blame] | 5813 | Variable *SP = getPhysicalRegister(RegARM32::Reg_sp); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5814 | Context.insert<InstFakeUse>(SP); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5815 | } |
| 5816 | |
John Porto | a47c11c | 2016-04-21 05:53:42 -0700 | [diff] [blame] | 5817 | void TargetARM32::lowerShuffleVector(const InstShuffleVector *Instr) { |
| 5818 | auto *Dest = Instr->getDest(); |
| 5819 | const Type DestTy = Dest->getType(); |
| 5820 | |
| 5821 | auto *T = makeReg(DestTy); |
| 5822 | |
| 5823 | switch (DestTy) { |
| 5824 | default: |
| 5825 | break; |
| 5826 | // TODO(jpp): figure out how to properly lower this without scalarization. |
| 5827 | } |
| 5828 | |
| 5829 | // Unoptimized shuffle. Perform a series of inserts and extracts. |
| 5830 | Context.insert<InstFakeDef>(T); |
| 5831 | auto *Src0 = llvm::cast<Variable>(Instr->getSrc(0)); |
| 5832 | auto *Src1 = llvm::cast<Variable>(Instr->getSrc(1)); |
| 5833 | const SizeT NumElements = typeNumElements(DestTy); |
| 5834 | const Type ElementType = typeElementType(DestTy); |
| 5835 | for (SizeT I = 0; I < Instr->getNumIndexes(); ++I) { |
| 5836 | auto *Index = Instr->getIndex(I); |
| 5837 | const SizeT Elem = Index->getValue(); |
| 5838 | auto *ExtElmt = makeReg(ElementType); |
| 5839 | if (Elem < NumElements) { |
| 5840 | lowerExtractElement( |
| 5841 | InstExtractElement::create(Func, ExtElmt, Src0, Index)); |
| 5842 | } else { |
| 5843 | lowerExtractElement(InstExtractElement::create( |
| 5844 | Func, ExtElmt, Src1, |
| 5845 | Ctx->getConstantInt32(Index->getValue() - NumElements))); |
| 5846 | } |
| 5847 | auto *NewT = makeReg(DestTy); |
| 5848 | lowerInsertElement(InstInsertElement::create(Func, NewT, T, ExtElmt, |
| 5849 | Ctx->getConstantInt32(I))); |
| 5850 | T = NewT; |
| 5851 | } |
| 5852 | _mov(Dest, T); |
| 5853 | } |
| 5854 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5855 | void TargetARM32::lowerSelect(const InstSelect *Instr) { |
| 5856 | Variable *Dest = Instr->getDest(); |
Jan Voung | e0df91f | 2015-06-30 08:47:06 -0700 | [diff] [blame] | 5857 | Type DestTy = Dest->getType(); |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5858 | Operand *SrcT = Instr->getTrueOperand(); |
| 5859 | Operand *SrcF = Instr->getFalseOperand(); |
| 5860 | Operand *Condition = Instr->getCondition(); |
Jan Voung | e0df91f | 2015-06-30 08:47:06 -0700 | [diff] [blame] | 5861 | |
John Porto | 397f602 | 2016-04-15 06:26:58 -0700 | [diff] [blame] | 5862 | if (!isVectorType(DestTy)) { |
| 5863 | lowerInt1ForSelect(Dest, Condition, legalizeUndef(SrcT), |
| 5864 | legalizeUndef(SrcF)); |
Jan Voung | e0df91f | 2015-06-30 08:47:06 -0700 | [diff] [blame] | 5865 | return; |
| 5866 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 5867 | |
John Porto | 397f602 | 2016-04-15 06:26:58 -0700 | [diff] [blame] | 5868 | Type TType = DestTy; |
| 5869 | switch (DestTy) { |
| 5870 | default: |
| 5871 | llvm::report_fatal_error("Unexpected type for vector select."); |
| 5872 | case IceType_v4i1: |
| 5873 | TType = IceType_v4i32; |
| 5874 | break; |
| 5875 | case IceType_v8i1: |
| 5876 | TType = IceType_v8i16; |
| 5877 | break; |
| 5878 | case IceType_v16i1: |
| 5879 | TType = IceType_v16i8; |
| 5880 | break; |
| 5881 | case IceType_v4f32: |
| 5882 | TType = IceType_v4i32; |
| 5883 | break; |
| 5884 | case IceType_v4i32: |
| 5885 | case IceType_v8i16: |
| 5886 | case IceType_v16i8: |
| 5887 | break; |
| 5888 | } |
| 5889 | auto *T = makeReg(TType); |
| 5890 | lowerCast(InstCast::create(Func, InstCast::Sext, T, Condition)); |
| 5891 | auto *SrcTR = legalizeToReg(SrcT); |
| 5892 | auto *SrcFR = legalizeToReg(SrcF); |
| 5893 | _vbsl(T, SrcTR, SrcFR)->setDestRedefined(); |
| 5894 | _mov(Dest, T); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5895 | } |
| 5896 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5897 | void TargetARM32::lowerStore(const InstStore *Instr) { |
| 5898 | Operand *Value = Instr->getData(); |
| 5899 | Operand *Addr = Instr->getAddr(); |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 5900 | OperandARM32Mem *NewAddr = formMemoryOperand(Addr, Value->getType()); |
| 5901 | Type Ty = NewAddr->getType(); |
| 5902 | |
| 5903 | if (Ty == IceType_i64) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 5904 | Value = legalizeUndef(Value); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5905 | Variable *ValueHi = legalizeToReg(hiOperand(Value)); |
| 5906 | Variable *ValueLo = legalizeToReg(loOperand(Value)); |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 5907 | _str(ValueHi, llvm::cast<OperandARM32Mem>(hiOperand(NewAddr))); |
| 5908 | _str(ValueLo, llvm::cast<OperandARM32Mem>(loOperand(NewAddr))); |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 5909 | } else { |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5910 | Variable *ValueR = legalizeToReg(Value); |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 5911 | _str(ValueR, NewAddr); |
| 5912 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5913 | } |
| 5914 | |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5915 | void TargetARM32::doAddressOptStore() { |
Jim Stichnoth | f5fdd23 | 2016-05-09 12:24:36 -0700 | [diff] [blame] | 5916 | Inst *Instr = iteratorToInst(Context.getCur()); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5917 | assert(llvm::isa<InstStore>(Instr)); |
| 5918 | Operand *Src = Instr->getSrc(0); |
| 5919 | Operand *Addr = Instr->getSrc(1); |
| 5920 | if (OperandARM32Mem *Mem = |
| 5921 | formAddressingMode(Src->getType(), Func, Instr, Addr)) { |
| 5922 | Instr->setDeleted(); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 5923 | Context.insert<InstStore>(Src, Mem); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 5924 | } |
| 5925 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5926 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5927 | void TargetARM32::lowerSwitch(const InstSwitch *Instr) { |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5928 | // This implements the most naive possible lowering. |
| 5929 | // cmp a,val[0]; jeq label[0]; cmp a,val[1]; jeq label[1]; ... jmp default |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5930 | Operand *Src0 = Instr->getComparison(); |
| 5931 | SizeT NumCases = Instr->getNumCases(); |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5932 | if (Src0->getType() == IceType_i64) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 5933 | Src0 = legalizeUndef(Src0); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5934 | Variable *Src0Lo = legalizeToReg(loOperand(Src0)); |
| 5935 | Variable *Src0Hi = legalizeToReg(hiOperand(Src0)); |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5936 | for (SizeT I = 0; I < NumCases; ++I) { |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5937 | Operand *ValueLo = Ctx->getConstantInt32(Instr->getValue(I)); |
| 5938 | Operand *ValueHi = Ctx->getConstantInt32(Instr->getValue(I) >> 32); |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5939 | ValueLo = legalize(ValueLo, Legal_Reg | Legal_Flex); |
| 5940 | ValueHi = legalize(ValueHi, Legal_Reg | Legal_Flex); |
| 5941 | _cmp(Src0Lo, ValueLo); |
| 5942 | _cmp(Src0Hi, ValueHi, CondARM32::EQ); |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5943 | _br(Instr->getLabel(I), CondARM32::EQ); |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5944 | } |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5945 | _br(Instr->getLabelDefault()); |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5946 | return; |
| 5947 | } |
Jan Voung | e0df91f | 2015-06-30 08:47:06 -0700 | [diff] [blame] | 5948 | |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 5949 | Variable *Src0Var = legalizeToReg(Src0); |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 5950 | // If Src0 is not an i32, we left shift it -- see the icmp lowering for the |
| 5951 | // reason. |
| 5952 | assert(Src0Var->mustHaveReg()); |
| 5953 | const size_t ShiftAmt = 32 - getScalarIntBitWidth(Src0->getType()); |
| 5954 | assert(ShiftAmt < 32); |
| 5955 | if (ShiftAmt > 0) { |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 5956 | Operand *ShAmtImm = shAmtImm(ShiftAmt); |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 5957 | Variable *T = makeReg(IceType_i32); |
John Porto | 2758bb0 | 2015-11-17 14:31:25 -0800 | [diff] [blame] | 5958 | _lsl(T, Src0Var, ShAmtImm); |
John Porto | afc92af | 2015-10-16 10:34:04 -0700 | [diff] [blame] | 5959 | Src0Var = T; |
| 5960 | } |
| 5961 | |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5962 | for (SizeT I = 0; I < NumCases; ++I) { |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5963 | Operand *Value = Ctx->getConstantInt32(Instr->getValue(I) << ShiftAmt); |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5964 | Value = legalize(Value, Legal_Reg | Legal_Flex); |
| 5965 | _cmp(Src0Var, Value); |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5966 | _br(Instr->getLabel(I), CondARM32::EQ); |
Andrew Scull | fdc54db | 2015-06-29 11:21:18 -0700 | [diff] [blame] | 5967 | } |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5968 | _br(Instr->getLabelDefault()); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5969 | } |
| 5970 | |
Eric Holk | 67c7c41 | 2016-04-15 13:05:37 -0700 | [diff] [blame] | 5971 | void TargetARM32::lowerBreakpoint(const InstBreakpoint *Instr) { |
| 5972 | UnimplementedLoweringError(this, Instr); |
| 5973 | } |
| 5974 | |
Jim Stichnoth | 8cfeb69 | 2016-02-05 09:50:02 -0800 | [diff] [blame] | 5975 | void TargetARM32::lowerUnreachable(const InstUnreachable * /*Instr*/) { |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 5976 | _trap(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 5977 | } |
| 5978 | |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 5979 | namespace { |
| 5980 | // Returns whether Opnd needs the GOT address. Currently, ConstantRelocatables, |
| 5981 | // and fp constants will need access to the GOT address. |
| 5982 | bool operandNeedsGot(const Operand *Opnd) { |
| 5983 | if (llvm::isa<ConstantRelocatable>(Opnd)) { |
| 5984 | return true; |
| 5985 | } |
| 5986 | |
| 5987 | if (llvm::isa<ConstantFloat>(Opnd)) { |
| 5988 | uint32_t _; |
| 5989 | return !OperandARM32FlexFpImm::canHoldImm(Opnd, &_); |
| 5990 | } |
| 5991 | |
| 5992 | const auto *F64 = llvm::dyn_cast<ConstantDouble>(Opnd); |
| 5993 | if (F64 != nullptr) { |
| 5994 | uint32_t _; |
| 5995 | return !OperandARM32FlexFpImm::canHoldImm(Opnd, &_) && |
| 5996 | !isFloatingPointZero(F64); |
| 5997 | } |
| 5998 | |
| 5999 | return false; |
| 6000 | } |
| 6001 | |
| 6002 | // Returns whether Phi needs the GOT address (which it does if any of its |
| 6003 | // operands needs the GOT address.) |
| 6004 | bool phiNeedsGot(const InstPhi *Phi) { |
| 6005 | if (Phi->isDeleted()) { |
| 6006 | return false; |
| 6007 | } |
| 6008 | |
| 6009 | for (SizeT I = 0; I < Phi->getSrcSize(); ++I) { |
| 6010 | if (operandNeedsGot(Phi->getSrc(I))) { |
| 6011 | return true; |
| 6012 | } |
| 6013 | } |
| 6014 | |
| 6015 | return false; |
| 6016 | } |
| 6017 | |
| 6018 | // Returns whether **any** phi in Node needs the GOT address. |
| 6019 | bool anyPhiInNodeNeedsGot(CfgNode *Node) { |
| 6020 | for (auto &Inst : Node->getPhis()) { |
| 6021 | if (phiNeedsGot(llvm::cast<InstPhi>(&Inst))) { |
| 6022 | return true; |
| 6023 | } |
| 6024 | } |
| 6025 | return false; |
| 6026 | } |
| 6027 | |
| 6028 | } // end of anonymous namespace |
| 6029 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6030 | void TargetARM32::prelowerPhis() { |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 6031 | CfgNode *Node = Context.getNode(); |
| 6032 | |
| 6033 | if (SandboxingType == ST_Nonsfi) { |
| 6034 | assert(GotPtr != nullptr); |
| 6035 | if (anyPhiInNodeNeedsGot(Node)) { |
| 6036 | // If any phi instruction needs the GOT address, we place a |
| 6037 | // fake-use GotPtr |
| 6038 | // in Node to prevent the GotPtr's initialization from being dead code |
| 6039 | // eliminated. |
| 6040 | Node->getInsts().push_front(InstFakeUse::create(Func, GotPtr)); |
| 6041 | } |
| 6042 | } |
| 6043 | |
| 6044 | PhiLowering::prelowerPhis32Bit(this, Node, Func); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6045 | } |
| 6046 | |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6047 | Variable *TargetARM32::makeVectorOfZeros(Type Ty, RegNumT RegNum) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6048 | Variable *Reg = makeReg(Ty, RegNum); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 6049 | Context.insert<InstFakeDef>(Reg); |
Eric Holk | cfc2553 | 2016-02-09 17:47:58 -0800 | [diff] [blame] | 6050 | assert(isVectorType(Ty)); |
| 6051 | _veor(Reg, Reg, Reg); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6052 | return Reg; |
| 6053 | } |
| 6054 | |
| 6055 | // Helper for legalize() to emit the right code to lower an operand to a |
| 6056 | // register of the appropriate type. |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6057 | Variable *TargetARM32::copyToReg(Operand *Src, RegNumT RegNum) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6058 | Type Ty = Src->getType(); |
| 6059 | Variable *Reg = makeReg(Ty, RegNum); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 6060 | if (auto *Mem = llvm::dyn_cast<OperandARM32Mem>(Src)) { |
| 6061 | _ldr(Reg, Mem); |
| 6062 | } else { |
| 6063 | _mov(Reg, Src); |
| 6064 | } |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6065 | return Reg; |
| 6066 | } |
| 6067 | |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6068 | // TODO(jpp): remove unneeded else clauses in legalize. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6069 | Operand *TargetARM32::legalize(Operand *From, LegalMask Allowed, |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6070 | RegNumT RegNum) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6071 | Type Ty = From->getType(); |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6072 | // Assert that a physical register is allowed. To date, all calls to |
| 6073 | // legalize() allow a physical register. Legal_Flex converts registers to the |
| 6074 | // right type OperandARM32FlexReg as needed. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6075 | assert(Allowed & Legal_Reg); |
John Porto | 562233c | 2015-10-28 05:47:58 -0700 | [diff] [blame] | 6076 | |
| 6077 | // Copied ipsis literis from TargetX86Base<Machine>. |
Reed Kotler | 5fa0a5f | 2016-02-15 20:01:24 -0800 | [diff] [blame] | 6078 | if (RegNum.hasNoValue()) { |
John Porto | 562233c | 2015-10-28 05:47:58 -0700 | [diff] [blame] | 6079 | if (Variable *Subst = getContext().availabilityGet(From)) { |
| 6080 | // At this point we know there is a potential substitution available. |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 6081 | if (!Subst->isRematerializable() && Subst->mustHaveReg() && |
| 6082 | !Subst->hasReg()) { |
John Porto | 562233c | 2015-10-28 05:47:58 -0700 | [diff] [blame] | 6083 | // At this point we know the substitution will have a register. |
| 6084 | if (From->getType() == Subst->getType()) { |
| 6085 | // At this point we know the substitution's register is compatible. |
| 6086 | return Subst; |
| 6087 | } |
| 6088 | } |
| 6089 | } |
| 6090 | } |
| 6091 | |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6092 | // Go through the various types of operands: OperandARM32Mem, |
| 6093 | // OperandARM32Flex, Constant, and Variable. Given the above assertion, if |
| 6094 | // type of operand is not legal (e.g., OperandARM32Mem and !Legal_Mem), we |
| 6095 | // can always copy to a register. |
Jim Stichnoth | 5bff61c | 2015-10-28 09:26:00 -0700 | [diff] [blame] | 6096 | if (auto *Mem = llvm::dyn_cast<OperandARM32Mem>(From)) { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6097 | // Before doing anything with a Mem operand, we need to ensure that the |
| 6098 | // Base and Index components are in physical registers. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6099 | Variable *Base = Mem->getBase(); |
| 6100 | Variable *Index = Mem->getIndex(); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 6101 | ConstantInteger32 *Offset = Mem->getOffset(); |
| 6102 | assert(Index == nullptr || Offset == nullptr); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6103 | Variable *RegBase = nullptr; |
| 6104 | Variable *RegIndex = nullptr; |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 6105 | assert(Base); |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 6106 | RegBase = llvm::cast<Variable>( |
| 6107 | legalize(Base, Legal_Reg | Legal_Rematerializable)); |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 6108 | assert(Ty < MemTraitsSize); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6109 | if (Index) { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 6110 | assert(Offset == nullptr); |
| 6111 | assert(MemTraits[Ty].CanHaveIndex); |
Andrew Scull | 97f460d | 2015-07-21 10:07:42 -0700 | [diff] [blame] | 6112 | RegIndex = legalizeToReg(Index); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6113 | } |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 6114 | if (Offset && Offset->getValue() != 0) { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 6115 | assert(Index == nullptr); |
| 6116 | static constexpr bool ZeroExt = false; |
| 6117 | assert(MemTraits[Ty].CanHaveImm); |
| 6118 | if (!OperandARM32Mem::canHoldOffset(Ty, ZeroExt, Offset->getValue())) { |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 6119 | llvm::report_fatal_error("Invalid memory offset."); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 6120 | } |
| 6121 | } |
| 6122 | |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6123 | // Create a new operand if there was a change. |
| 6124 | if (Base != RegBase || Index != RegIndex) { |
| 6125 | // There is only a reg +/- reg or reg + imm form. |
| 6126 | // Figure out which to re-create. |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 6127 | if (RegIndex) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6128 | Mem = OperandARM32Mem::create(Func, Ty, RegBase, RegIndex, |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6129 | Mem->getShiftOp(), Mem->getShiftAmt(), |
| 6130 | Mem->getAddrMode()); |
| 6131 | } else { |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 6132 | Mem = OperandARM32Mem::create(Func, Ty, RegBase, Offset, |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6133 | Mem->getAddrMode()); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6134 | } |
| 6135 | } |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 6136 | if (Allowed & Legal_Mem) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6137 | From = Mem; |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 6138 | } else { |
| 6139 | Variable *Reg = makeReg(Ty, RegNum); |
John Porto | 3f6b47d | 2015-11-19 05:42:59 -0800 | [diff] [blame] | 6140 | _ldr(Reg, Mem); |
John Porto | ba6a67c | 2015-09-25 15:19:45 -0700 | [diff] [blame] | 6141 | From = Reg; |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6142 | } |
| 6143 | return From; |
| 6144 | } |
| 6145 | |
Jim Stichnoth | 5bff61c | 2015-10-28 09:26:00 -0700 | [diff] [blame] | 6146 | if (auto *Flex = llvm::dyn_cast<OperandARM32Flex>(From)) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6147 | if (!(Allowed & Legal_Flex)) { |
Jim Stichnoth | 5bff61c | 2015-10-28 09:26:00 -0700 | [diff] [blame] | 6148 | if (auto *FlexReg = llvm::dyn_cast<OperandARM32FlexReg>(Flex)) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6149 | if (FlexReg->getShiftOp() == OperandARM32::kNoShift) { |
| 6150 | From = FlexReg->getReg(); |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6151 | // Fall through and let From be checked as a Variable below, where it |
| 6152 | // may or may not need a register. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6153 | } else { |
| 6154 | return copyToReg(Flex, RegNum); |
| 6155 | } |
| 6156 | } else { |
| 6157 | return copyToReg(Flex, RegNum); |
| 6158 | } |
| 6159 | } else { |
| 6160 | return From; |
| 6161 | } |
| 6162 | } |
| 6163 | |
| 6164 | if (llvm::isa<Constant>(From)) { |
| 6165 | if (llvm::isa<ConstantUndef>(From)) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6166 | From = legalizeUndef(From, RegNum); |
| 6167 | if (isVectorType(Ty)) |
| 6168 | return From; |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6169 | } |
| 6170 | // There should be no constants of vector type (other than undef). |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6171 | assert(!isVectorType(Ty)); |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6172 | if (auto *C32 = llvm::dyn_cast<ConstantInteger32>(From)) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6173 | uint32_t RotateAmt; |
| 6174 | uint32_t Immed_8; |
| 6175 | uint32_t Value = static_cast<uint32_t>(C32->getValue()); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6176 | if (OperandARM32FlexImm::canHoldImm(Value, &RotateAmt, &Immed_8)) { |
| 6177 | // The immediate can be encoded as a Flex immediate. We may return the |
| 6178 | // Flex operand if the caller has Allow'ed it. |
| 6179 | auto *OpF = OperandARM32FlexImm::create(Func, Ty, Immed_8, RotateAmt); |
| 6180 | const bool CanBeFlex = Allowed & Legal_Flex; |
| 6181 | if (CanBeFlex) |
| 6182 | return OpF; |
| 6183 | return copyToReg(OpF, RegNum); |
| 6184 | } else if (OperandARM32FlexImm::canHoldImm(~Value, &RotateAmt, |
| 6185 | &Immed_8)) { |
| 6186 | // Even though the immediate can't be encoded as a Flex operand, its |
| 6187 | // inverted bit pattern can, thus we use ARM's mvn to load the 32-bit |
| 6188 | // constant with a single instruction. |
| 6189 | auto *InvOpF = |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6190 | OperandARM32FlexImm::create(Func, Ty, Immed_8, RotateAmt); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6191 | Variable *Reg = makeReg(Ty, RegNum); |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6192 | _mvn(Reg, InvOpF); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6193 | return Reg; |
| 6194 | } else { |
| 6195 | // Do a movw/movt to a register. |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6196 | Variable *Reg = makeReg(Ty, RegNum); |
| 6197 | uint32_t UpperBits = (Value >> 16) & 0xFFFF; |
| 6198 | _movw(Reg, |
| 6199 | UpperBits != 0 ? Ctx->getConstantInt32(Value & 0xFFFF) : C32); |
| 6200 | if (UpperBits != 0) { |
| 6201 | _movt(Reg, Ctx->getConstantInt32(UpperBits)); |
| 6202 | } |
| 6203 | return Reg; |
| 6204 | } |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6205 | } else if (auto *C = llvm::dyn_cast<ConstantRelocatable>(From)) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6206 | Variable *Reg = makeReg(Ty, RegNum); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 6207 | if (SandboxingType != ST_Nonsfi) { |
| 6208 | _movw(Reg, C); |
| 6209 | _movt(Reg, C); |
| 6210 | } else { |
| 6211 | auto *GotAddr = legalizeToReg(GotPtr); |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 6212 | GlobalString CGotoffName = createGotoffRelocation(C); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 6213 | loadNamedConstantRelocatablePIC( |
| 6214 | CGotoffName, Reg, [this, Reg](Variable *PC) { |
| 6215 | _ldr(Reg, OperandARM32Mem::create(Func, IceType_i32, PC, Reg)); |
| 6216 | }); |
| 6217 | _add(Reg, GotAddr, Reg); |
| 6218 | } |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6219 | return Reg; |
| 6220 | } else { |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 6221 | assert(isScalarFloatingType(Ty)); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 6222 | uint32_t ModifiedImm; |
| 6223 | if (OperandARM32FlexFpImm::canHoldImm(From, &ModifiedImm)) { |
| 6224 | Variable *T = makeReg(Ty, RegNum); |
| 6225 | _mov(T, |
| 6226 | OperandARM32FlexFpImm::create(Func, From->getType(), ModifiedImm)); |
| 6227 | return T; |
| 6228 | } |
| 6229 | |
| 6230 | if (Ty == IceType_f64 && isFloatingPointZero(From)) { |
| 6231 | // Use T = T ^ T to load a 64-bit fp zero. This does not work for f32 |
| 6232 | // because ARM does not have a veor instruction with S registers. |
| 6233 | Variable *T = makeReg(IceType_f64, RegNum); |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 6234 | Context.insert<InstFakeDef>(T); |
John Porto | ccea793 | 2015-11-17 04:58:36 -0800 | [diff] [blame] | 6235 | _veor(T, T, T); |
| 6236 | return T; |
| 6237 | } |
| 6238 | |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6239 | // Load floats/doubles from literal pool. |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 6240 | auto *CFrom = llvm::cast<Constant>(From); |
| 6241 | assert(CFrom->getShouldBePooled()); |
| 6242 | Constant *Offset = Ctx->getConstantSym(0, CFrom->getLabelName()); |
John Porto | dc61925 | 2016-02-10 15:57:16 -0800 | [diff] [blame] | 6243 | Variable *BaseReg = nullptr; |
| 6244 | if (SandboxingType == ST_Nonsfi) { |
| 6245 | // vldr does not support the [base, index] addressing mode, so we need |
| 6246 | // to legalize Offset to a register. Otherwise, we could simply |
| 6247 | // vldr dest, [got, reg(Offset)] |
| 6248 | BaseReg = legalizeToReg(Offset); |
| 6249 | } else { |
| 6250 | BaseReg = makeReg(getPointerType()); |
| 6251 | _movw(BaseReg, Offset); |
| 6252 | _movt(BaseReg, Offset); |
| 6253 | } |
Jan Voung | 86ebec1 | 2015-08-09 07:58:35 -0700 | [diff] [blame] | 6254 | From = formMemoryOperand(BaseReg, Ty); |
| 6255 | return copyToReg(From, RegNum); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6256 | } |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6257 | } |
| 6258 | |
Jim Stichnoth | 5bff61c | 2015-10-28 09:26:00 -0700 | [diff] [blame] | 6259 | if (auto *Var = llvm::dyn_cast<Variable>(From)) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 6260 | if (Var->isRematerializable()) { |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 6261 | if (Allowed & Legal_Rematerializable) { |
| 6262 | return From; |
| 6263 | } |
| 6264 | |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 6265 | Variable *T = makeReg(Var->getType(), RegNum); |
| 6266 | _mov(T, Var); |
| 6267 | return T; |
| 6268 | } |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6269 | // Check if the variable is guaranteed a physical register. This can happen |
| 6270 | // either when the variable is pre-colored or when it is assigned infinite |
| 6271 | // weight. |
Andrew Scull | 11c9a32 | 2015-08-28 14:24:14 -0700 | [diff] [blame] | 6272 | bool MustHaveRegister = (Var->hasReg() || Var->mustHaveReg()); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6273 | // We need a new physical register for the operand if: |
| 6274 | // Mem is not allowed and Var isn't guaranteed a physical |
| 6275 | // register, or |
| 6276 | // RegNum is required and Var->getRegNum() doesn't match. |
| 6277 | if ((!(Allowed & Legal_Mem) && !MustHaveRegister) || |
Reed Kotler | 5fa0a5f | 2016-02-15 20:01:24 -0800 | [diff] [blame] | 6278 | (RegNum.hasValue() && (RegNum != Var->getRegNum()))) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6279 | From = copyToReg(From, RegNum); |
| 6280 | } |
| 6281 | return From; |
| 6282 | } |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 6283 | llvm::report_fatal_error("Unhandled operand kind in legalize()"); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6284 | |
| 6285 | return From; |
| 6286 | } |
| 6287 | |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6288 | /// Provide a trivial wrapper to legalize() for this common usage. |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6289 | Variable *TargetARM32::legalizeToReg(Operand *From, RegNumT RegNum) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6290 | return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); |
| 6291 | } |
| 6292 | |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6293 | /// Legalize undef values to concrete values. |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6294 | Operand *TargetARM32::legalizeUndef(Operand *From, RegNumT RegNum) { |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6295 | Type Ty = From->getType(); |
| 6296 | if (llvm::isa<ConstantUndef>(From)) { |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6297 | // Lower undefs to zero. Another option is to lower undefs to an |
| 6298 | // uninitialized register; however, using an uninitialized register results |
| 6299 | // in less predictable code. |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6300 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6301 | // If in the future the implementation is changed to lower undef values to |
| 6302 | // uninitialized registers, a FakeDef will be needed: |
| 6303 | // Context.insert(InstFakeDef::create(Func, Reg)); This is in order to |
| 6304 | // ensure that the live range of Reg is not overestimated. If the constant |
| 6305 | // being lowered is a 64 bit value, then the result should be split and the |
| 6306 | // lo and hi components will need to go in uninitialized registers. |
Jan Voung | fbdd244 | 2015-07-15 12:36:20 -0700 | [diff] [blame] | 6307 | if (isVectorType(Ty)) |
| 6308 | return makeVectorOfZeros(Ty, RegNum); |
| 6309 | return Ctx->getConstantZero(Ty); |
| 6310 | } |
| 6311 | return From; |
| 6312 | } |
| 6313 | |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 6314 | OperandARM32Mem *TargetARM32::formMemoryOperand(Operand *Operand, Type Ty) { |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 6315 | auto *Mem = llvm::dyn_cast<OperandARM32Mem>(Operand); |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6316 | // It may be the case that address mode optimization already creates an |
| 6317 | // OperandARM32Mem, so in that case it wouldn't need another level of |
| 6318 | // transformation. |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 6319 | if (Mem) { |
| 6320 | return llvm::cast<OperandARM32Mem>(legalize(Mem)); |
| 6321 | } |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 6322 | // If we didn't do address mode optimization, then we only have a |
| 6323 | // base/offset to work with. ARM always requires a base register, so |
| 6324 | // just use that to hold the operand. |
Jim Stichnoth | 54f3d51 | 2015-12-11 09:53:00 -0800 | [diff] [blame] | 6325 | auto *Base = llvm::cast<Variable>( |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 6326 | legalize(Operand, Legal_Reg | Legal_Rematerializable)); |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 6327 | return OperandARM32Mem::create( |
John Porto | 866b6b1 | 2015-12-03 09:45:31 -0800 | [diff] [blame] | 6328 | Func, Ty, Base, |
Jan Voung | befd03a | 2015-06-02 11:03:03 -0700 | [diff] [blame] | 6329 | llvm::cast<ConstantInteger32>(Ctx->getConstantZero(IceType_i32))); |
| 6330 | } |
| 6331 | |
John Porto | 578f116 | 2015-10-06 06:54:42 -0700 | [diff] [blame] | 6332 | Variable64On32 *TargetARM32::makeI64RegPair() { |
| 6333 | Variable64On32 *Reg = |
| 6334 | llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64)); |
| 6335 | Reg->setMustHaveReg(); |
| 6336 | Reg->initHiLo(Func); |
| 6337 | Reg->getLo()->setMustNotHaveReg(); |
| 6338 | Reg->getHi()->setMustNotHaveReg(); |
| 6339 | return Reg; |
| 6340 | } |
| 6341 | |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6342 | Variable *TargetARM32::makeReg(Type Type, RegNumT RegNum) { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6343 | // There aren't any 64-bit integer registers for ARM32. |
| 6344 | assert(Type != IceType_i64); |
Reed Kotler | 5fa0a5f | 2016-02-15 20:01:24 -0800 | [diff] [blame] | 6345 | assert(AllowTemporaryWithNoReg || RegNum.hasValue()); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6346 | Variable *Reg = Func->makeVariable(Type); |
Reed Kotler | 5fa0a5f | 2016-02-15 20:01:24 -0800 | [diff] [blame] | 6347 | if (RegNum.hasValue()) |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6348 | Reg->setRegNum(RegNum); |
Reed Kotler | 5fa0a5f | 2016-02-15 20:01:24 -0800 | [diff] [blame] | 6349 | else |
| 6350 | Reg->setMustHaveReg(); |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6351 | return Reg; |
| 6352 | } |
| 6353 | |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 6354 | void TargetARM32::alignRegisterPow2(Variable *Reg, uint32_t Align, |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6355 | RegNumT TmpRegNum) { |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 6356 | assert(llvm::isPowerOf2_32(Align)); |
Jan Voung | 0fa6c5a | 2015-06-01 11:04:04 -0700 | [diff] [blame] | 6357 | uint32_t RotateAmt; |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 6358 | uint32_t Immed_8; |
| 6359 | Operand *Mask; |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 6360 | // Use AND or BIC to mask off the bits, depending on which immediate fits (if |
| 6361 | // it fits at all). Assume Align is usually small, in which case BIC works |
| 6362 | // better. Thus, this rounds down to the alignment. |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 6363 | if (OperandARM32FlexImm::canHoldImm(Align - 1, &RotateAmt, &Immed_8)) { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 6364 | Mask = legalize(Ctx->getConstantInt32(Align - 1), Legal_Reg | Legal_Flex, |
| 6365 | TmpRegNum); |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 6366 | _bic(Reg, Reg, Mask); |
| 6367 | } else { |
John Porto | 614140e | 2015-11-23 11:43:13 -0800 | [diff] [blame] | 6368 | Mask = legalize(Ctx->getConstantInt32(-Align), Legal_Reg | Legal_Flex, |
| 6369 | TmpRegNum); |
Jan Voung | 55500db | 2015-05-26 14:25:40 -0700 | [diff] [blame] | 6370 | _and(Reg, Reg, Mask); |
| 6371 | } |
| 6372 | } |
| 6373 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6374 | void TargetARM32::postLower() { |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 6375 | if (getFlags().getOptLevel() == Opt_m1) |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6376 | return; |
Jim Stichnoth | 230d410 | 2015-09-25 17:40:32 -0700 | [diff] [blame] | 6377 | markRedefinitions(); |
John Porto | 562233c | 2015-10-28 05:47:58 -0700 | [diff] [blame] | 6378 | Context.availabilityUpdate(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6379 | } |
| 6380 | |
| 6381 | void TargetARM32::makeRandomRegisterPermutation( |
Jim Stichnoth | 8aa3966 | 2016-02-10 11:20:30 -0800 | [diff] [blame] | 6382 | llvm::SmallVectorImpl<RegNumT> &Permutation, |
John Porto | e82b560 | 2016-02-24 15:58:55 -0800 | [diff] [blame] | 6383 | const SmallBitVector &ExcludeRegisters, uint64_t Salt) const { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6384 | (void)Permutation; |
| 6385 | (void)ExcludeRegisters; |
Qining Lu | aee5fa8 | 2015-08-20 14:59:03 -0700 | [diff] [blame] | 6386 | (void)Salt; |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 6387 | UnimplementedError(getFlags()); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6388 | } |
| 6389 | |
Jan Voung | 76bb0be | 2015-05-14 09:26:19 -0700 | [diff] [blame] | 6390 | void TargetARM32::emit(const ConstantInteger32 *C) const { |
Jim Stichnoth | 20b71f5 | 2015-06-24 15:52:24 -0700 | [diff] [blame] | 6391 | if (!BuildDefs::dump()) |
Jan Voung | 76bb0be | 2015-05-14 09:26:19 -0700 | [diff] [blame] | 6392 | return; |
| 6393 | Ostream &Str = Ctx->getStrEmit(); |
Jim Stichnoth | 8ff4b28 | 2016-01-04 15:39:06 -0800 | [diff] [blame] | 6394 | Str << "#" << C->getValue(); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6395 | } |
| 6396 | |
Jan Voung | 76bb0be | 2015-05-14 09:26:19 -0700 | [diff] [blame] | 6397 | void TargetARM32::emit(const ConstantInteger64 *) const { |
| 6398 | llvm::report_fatal_error("Not expecting to emit 64-bit integers"); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6399 | } |
Jan Voung | 76bb0be | 2015-05-14 09:26:19 -0700 | [diff] [blame] | 6400 | |
| 6401 | void TargetARM32::emit(const ConstantFloat *C) const { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6402 | (void)C; |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 6403 | UnimplementedError(getFlags()); |
Jan Voung | 76bb0be | 2015-05-14 09:26:19 -0700 | [diff] [blame] | 6404 | } |
| 6405 | |
| 6406 | void TargetARM32::emit(const ConstantDouble *C) const { |
Jan Voung | b3401d2 | 2015-05-18 09:38:21 -0700 | [diff] [blame] | 6407 | (void)C; |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 6408 | UnimplementedError(getFlags()); |
Jan Voung | 76bb0be | 2015-05-14 09:26:19 -0700 | [diff] [blame] | 6409 | } |
| 6410 | |
| 6411 | void TargetARM32::emit(const ConstantUndef *) const { |
| 6412 | llvm::report_fatal_error("undef value encountered by emitter."); |
| 6413 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6414 | |
Jim Stichnoth | 8ff4b28 | 2016-01-04 15:39:06 -0800 | [diff] [blame] | 6415 | void TargetARM32::emit(const ConstantRelocatable *C) const { |
| 6416 | if (!BuildDefs::dump()) |
| 6417 | return; |
| 6418 | Ostream &Str = Ctx->getStrEmit(); |
| 6419 | Str << "#"; |
| 6420 | emitWithoutPrefix(C); |
| 6421 | } |
| 6422 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6423 | void TargetARM32::lowerInt1ForSelect(Variable *Dest, Operand *Boolean, |
| 6424 | Operand *TrueValue, Operand *FalseValue) { |
| 6425 | Operand *_1 = legalize(Ctx->getConstantInt1(1), Legal_Reg | Legal_Flex); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6426 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6427 | assert(Boolean->getType() == IceType_i1); |
| 6428 | |
| 6429 | bool NeedsAnd1 = false; |
| 6430 | if (TrueValue->getType() == IceType_i1) { |
| 6431 | assert(FalseValue->getType() == IceType_i1); |
| 6432 | |
| 6433 | Variable *TrueValueV = Func->makeVariable(IceType_i1); |
| 6434 | SafeBoolChain Src0Safe = lowerInt1(TrueValueV, TrueValue); |
| 6435 | TrueValue = TrueValueV; |
| 6436 | |
| 6437 | Variable *FalseValueV = Func->makeVariable(IceType_i1); |
| 6438 | SafeBoolChain Src1Safe = lowerInt1(FalseValueV, FalseValue); |
| 6439 | FalseValue = FalseValueV; |
| 6440 | |
| 6441 | NeedsAnd1 = Src0Safe == SBC_No || Src1Safe == SBC_No; |
| 6442 | } |
| 6443 | |
| 6444 | Variable *DestLo = (Dest->getType() == IceType_i64) |
| 6445 | ? llvm::cast<Variable>(loOperand(Dest)) |
| 6446 | : Dest; |
| 6447 | Variable *DestHi = (Dest->getType() == IceType_i64) |
| 6448 | ? llvm::cast<Variable>(hiOperand(Dest)) |
| 6449 | : nullptr; |
| 6450 | Operand *FalseValueLo = (FalseValue->getType() == IceType_i64) |
| 6451 | ? loOperand(FalseValue) |
| 6452 | : FalseValue; |
| 6453 | Operand *FalseValueHi = |
| 6454 | (FalseValue->getType() == IceType_i64) ? hiOperand(FalseValue) : nullptr; |
| 6455 | |
| 6456 | Operand *TrueValueLo = |
| 6457 | (TrueValue->getType() == IceType_i64) ? loOperand(TrueValue) : TrueValue; |
| 6458 | Operand *TrueValueHi = |
| 6459 | (TrueValue->getType() == IceType_i64) ? hiOperand(TrueValue) : nullptr; |
| 6460 | |
| 6461 | Variable *T_Lo = makeReg(DestLo->getType()); |
| 6462 | Variable *T_Hi = (DestHi == nullptr) ? nullptr : makeReg(DestHi->getType()); |
| 6463 | |
| 6464 | _mov(T_Lo, legalize(FalseValueLo, Legal_Reg | Legal_Flex)); |
| 6465 | if (DestHi) { |
| 6466 | _mov(T_Hi, legalize(FalseValueHi, Legal_Reg | Legal_Flex)); |
| 6467 | } |
| 6468 | |
| 6469 | CondWhenTrue Cond(CondARM32::kNone); |
| 6470 | // FlagsWereSet is used to determine wether Boolean was folded or not. If not, |
| 6471 | // add an explicit _tst instruction below. |
| 6472 | bool FlagsWereSet = false; |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6473 | if (const Inst *Producer = Computations.getProducerOf(Boolean)) { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6474 | switch (Producer->getKind()) { |
| 6475 | default: |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 6476 | llvm::report_fatal_error("Unexpected producer."); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6477 | case Inst::Icmp: { |
| 6478 | Cond = lowerIcmpCond(llvm::cast<InstIcmp>(Producer)); |
| 6479 | FlagsWereSet = true; |
| 6480 | } break; |
| 6481 | case Inst::Fcmp: { |
| 6482 | Cond = lowerFcmpCond(llvm::cast<InstFcmp>(Producer)); |
| 6483 | FlagsWereSet = true; |
| 6484 | } break; |
| 6485 | case Inst::Cast: { |
| 6486 | const auto *CastProducer = llvm::cast<InstCast>(Producer); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6487 | assert(CastProducer->getCastKind() == InstCast::Trunc); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6488 | Boolean = CastProducer->getSrc(0); |
| 6489 | // No flags were set, so a _tst(Src, 1) will be emitted below. Don't |
| 6490 | // bother legalizing Src to a Reg because it will be legalized before |
| 6491 | // emitting the tst instruction. |
| 6492 | FlagsWereSet = false; |
| 6493 | } break; |
| 6494 | case Inst::Arithmetic: { |
| 6495 | // This is a special case: we eagerly assumed Producer could be folded, |
| 6496 | // but in reality, it can't. No reason to panic: we just lower it using |
| 6497 | // the regular lowerArithmetic helper. |
| 6498 | const auto *ArithProducer = llvm::cast<InstArithmetic>(Producer); |
| 6499 | lowerArithmetic(ArithProducer); |
| 6500 | Boolean = ArithProducer->getDest(); |
| 6501 | // No flags were set, so a _tst(Dest, 1) will be emitted below. Don't |
| 6502 | // bother legalizing Dest to a Reg because it will be legalized before |
| 6503 | // emitting the tst instruction. |
| 6504 | FlagsWereSet = false; |
| 6505 | } break; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6506 | } |
| 6507 | } |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6508 | |
| 6509 | if (!FlagsWereSet) { |
| 6510 | // No flags have been set, so emit a tst Boolean, 1. |
| 6511 | Variable *Src = legalizeToReg(Boolean); |
| 6512 | _tst(Src, _1); |
| 6513 | Cond = CondWhenTrue(CondARM32::NE); // i.e., CondARM32::NotZero. |
| 6514 | } |
| 6515 | |
| 6516 | if (Cond.WhenTrue0 == CondARM32::kNone) { |
| 6517 | assert(Cond.WhenTrue1 == CondARM32::kNone); |
| 6518 | } else { |
| 6519 | _mov_redefined(T_Lo, legalize(TrueValueLo, Legal_Reg | Legal_Flex), |
| 6520 | Cond.WhenTrue0); |
| 6521 | if (DestHi) { |
| 6522 | _mov_redefined(T_Hi, legalize(TrueValueHi, Legal_Reg | Legal_Flex), |
| 6523 | Cond.WhenTrue0); |
| 6524 | } |
| 6525 | } |
| 6526 | |
| 6527 | if (Cond.WhenTrue1 != CondARM32::kNone) { |
| 6528 | _mov_redefined(T_Lo, legalize(TrueValueLo, Legal_Reg | Legal_Flex), |
| 6529 | Cond.WhenTrue1); |
| 6530 | if (DestHi) { |
| 6531 | _mov_redefined(T_Hi, legalize(TrueValueHi, Legal_Reg | Legal_Flex), |
| 6532 | Cond.WhenTrue1); |
| 6533 | } |
| 6534 | } |
| 6535 | |
| 6536 | if (NeedsAnd1) { |
| 6537 | // We lowered something that is unsafe (i.e., can't provably be zero or |
| 6538 | // one). Truncate the result. |
| 6539 | _and(T_Lo, T_Lo, _1); |
| 6540 | } |
| 6541 | |
| 6542 | _mov(DestLo, T_Lo); |
| 6543 | if (DestHi) { |
| 6544 | _mov(DestHi, T_Hi); |
| 6545 | } |
| 6546 | } |
| 6547 | |
| 6548 | TargetARM32::SafeBoolChain TargetARM32::lowerInt1(Variable *Dest, |
| 6549 | Operand *Boolean) { |
| 6550 | assert(Boolean->getType() == IceType_i1); |
| 6551 | Variable *T = makeReg(IceType_i1); |
| 6552 | Operand *_0 = |
| 6553 | legalize(Ctx->getConstantZero(IceType_i1), Legal_Reg | Legal_Flex); |
| 6554 | Operand *_1 = legalize(Ctx->getConstantInt1(1), Legal_Reg | Legal_Flex); |
| 6555 | |
| 6556 | SafeBoolChain Safe = SBC_Yes; |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6557 | if (const Inst *Producer = Computations.getProducerOf(Boolean)) { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6558 | switch (Producer->getKind()) { |
| 6559 | default: |
John Porto | c39ec10 | 2015-12-01 13:00:43 -0800 | [diff] [blame] | 6560 | llvm::report_fatal_error("Unexpected producer."); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6561 | case Inst::Icmp: { |
| 6562 | _mov(T, _0); |
| 6563 | CondWhenTrue Cond = lowerIcmpCond(llvm::cast<InstIcmp>(Producer)); |
| 6564 | assert(Cond.WhenTrue0 != CondARM32::AL); |
| 6565 | assert(Cond.WhenTrue0 != CondARM32::kNone); |
| 6566 | assert(Cond.WhenTrue1 == CondARM32::kNone); |
| 6567 | _mov_redefined(T, _1, Cond.WhenTrue0); |
| 6568 | } break; |
| 6569 | case Inst::Fcmp: { |
| 6570 | _mov(T, _0); |
| 6571 | Inst *MovZero = Context.getLastInserted(); |
| 6572 | CondWhenTrue Cond = lowerFcmpCond(llvm::cast<InstFcmp>(Producer)); |
| 6573 | if (Cond.WhenTrue0 == CondARM32::AL) { |
| 6574 | assert(Cond.WhenTrue1 == CondARM32::kNone); |
| 6575 | MovZero->setDeleted(); |
| 6576 | _mov(T, _1); |
| 6577 | } else if (Cond.WhenTrue0 != CondARM32::kNone) { |
| 6578 | _mov_redefined(T, _1, Cond.WhenTrue0); |
| 6579 | } |
| 6580 | if (Cond.WhenTrue1 != CondARM32::kNone) { |
| 6581 | assert(Cond.WhenTrue0 != CondARM32::kNone); |
| 6582 | assert(Cond.WhenTrue0 != CondARM32::AL); |
| 6583 | _mov_redefined(T, _1, Cond.WhenTrue1); |
| 6584 | } |
| 6585 | } break; |
| 6586 | case Inst::Cast: { |
| 6587 | const auto *CastProducer = llvm::cast<InstCast>(Producer); |
| 6588 | assert(CastProducer->getCastKind() == InstCast::Trunc); |
| 6589 | Operand *Src = CastProducer->getSrc(0); |
| 6590 | if (Src->getType() == IceType_i64) |
| 6591 | Src = loOperand(Src); |
| 6592 | _mov(T, legalize(Src, Legal_Reg | Legal_Flex)); |
| 6593 | Safe = SBC_No; |
| 6594 | } break; |
| 6595 | case Inst::Arithmetic: { |
| 6596 | const auto *ArithProducer = llvm::cast<InstArithmetic>(Producer); |
| 6597 | Safe = lowerInt1Arithmetic(ArithProducer); |
| 6598 | _mov(T, ArithProducer->getDest()); |
| 6599 | } break; |
| 6600 | } |
| 6601 | } else { |
| 6602 | _mov(T, legalize(Boolean, Legal_Reg | Legal_Flex)); |
| 6603 | } |
| 6604 | |
| 6605 | _mov(Dest, T); |
| 6606 | return Safe; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6607 | } |
| 6608 | |
| 6609 | namespace { |
| 6610 | namespace BoolFolding { |
| 6611 | bool shouldTrackProducer(const Inst &Instr) { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6612 | switch (Instr.getKind()) { |
| 6613 | default: |
| 6614 | return false; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6615 | case Inst::Icmp: |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6616 | case Inst::Fcmp: |
| 6617 | return true; |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6618 | case Inst::Cast: { |
| 6619 | switch (llvm::cast<InstCast>(&Instr)->getCastKind()) { |
| 6620 | default: |
| 6621 | return false; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6622 | case InstCast::Trunc: |
| 6623 | return true; |
| 6624 | } |
| 6625 | } |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6626 | case Inst::Arithmetic: { |
| 6627 | switch (llvm::cast<InstArithmetic>(&Instr)->getOp()) { |
| 6628 | default: |
| 6629 | return false; |
| 6630 | case InstArithmetic::And: |
| 6631 | case InstArithmetic::Or: |
| 6632 | return true; |
| 6633 | } |
| 6634 | } |
| 6635 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6636 | } |
| 6637 | |
| 6638 | bool isValidConsumer(const Inst &Instr) { |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6639 | switch (Instr.getKind()) { |
| 6640 | default: |
| 6641 | return false; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6642 | case Inst::Br: |
| 6643 | return true; |
| 6644 | case Inst::Select: |
| 6645 | return !isVectorType(Instr.getDest()->getType()); |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6646 | case Inst::Cast: { |
| 6647 | switch (llvm::cast<InstCast>(&Instr)->getCastKind()) { |
| 6648 | default: |
| 6649 | return false; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6650 | case InstCast::Sext: |
| 6651 | return !isVectorType(Instr.getDest()->getType()); |
| 6652 | case InstCast::Zext: |
| 6653 | return !isVectorType(Instr.getDest()->getType()); |
| 6654 | } |
| 6655 | } |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6656 | case Inst::Arithmetic: { |
| 6657 | switch (llvm::cast<InstArithmetic>(&Instr)->getOp()) { |
| 6658 | default: |
| 6659 | return false; |
| 6660 | case InstArithmetic::And: |
| 6661 | return !isVectorType(Instr.getDest()->getType()); |
| 6662 | case InstArithmetic::Or: |
| 6663 | return !isVectorType(Instr.getDest()->getType()); |
| 6664 | } |
| 6665 | } |
| 6666 | } |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6667 | } |
| 6668 | } // end of namespace BoolFolding |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6669 | |
| 6670 | namespace FpFolding { |
| 6671 | bool shouldTrackProducer(const Inst &Instr) { |
| 6672 | switch (Instr.getKind()) { |
| 6673 | default: |
| 6674 | return false; |
| 6675 | case Inst::Arithmetic: { |
| 6676 | switch (llvm::cast<InstArithmetic>(&Instr)->getOp()) { |
| 6677 | default: |
| 6678 | return false; |
| 6679 | case InstArithmetic::Fmul: |
| 6680 | return true; |
| 6681 | } |
| 6682 | } |
| 6683 | } |
| 6684 | } |
| 6685 | |
| 6686 | bool isValidConsumer(const Inst &Instr) { |
| 6687 | switch (Instr.getKind()) { |
| 6688 | default: |
| 6689 | return false; |
| 6690 | case Inst::Arithmetic: { |
| 6691 | switch (llvm::cast<InstArithmetic>(&Instr)->getOp()) { |
| 6692 | default: |
| 6693 | return false; |
| 6694 | case InstArithmetic::Fadd: |
| 6695 | case InstArithmetic::Fsub: |
| 6696 | return true; |
| 6697 | } |
| 6698 | } |
| 6699 | } |
| 6700 | } |
| 6701 | } // end of namespace FpFolding |
| 6702 | |
| 6703 | namespace IntFolding { |
| 6704 | bool shouldTrackProducer(const Inst &Instr) { |
| 6705 | switch (Instr.getKind()) { |
| 6706 | default: |
| 6707 | return false; |
| 6708 | case Inst::Arithmetic: { |
| 6709 | switch (llvm::cast<InstArithmetic>(&Instr)->getOp()) { |
| 6710 | default: |
| 6711 | return false; |
| 6712 | case InstArithmetic::Mul: |
| 6713 | return true; |
| 6714 | } |
| 6715 | } |
| 6716 | } |
| 6717 | } |
| 6718 | |
| 6719 | bool isValidConsumer(const Inst &Instr) { |
| 6720 | switch (Instr.getKind()) { |
| 6721 | default: |
| 6722 | return false; |
| 6723 | case Inst::Arithmetic: { |
| 6724 | switch (llvm::cast<InstArithmetic>(&Instr)->getOp()) { |
| 6725 | default: |
| 6726 | return false; |
| 6727 | case InstArithmetic::Add: |
| 6728 | case InstArithmetic::Sub: |
| 6729 | return true; |
| 6730 | } |
| 6731 | } |
| 6732 | } |
| 6733 | } |
| 6734 | } // end of namespace FpFolding |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6735 | } // end of anonymous namespace |
| 6736 | |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6737 | void TargetARM32::ComputationTracker::recordProducers(CfgNode *Node) { |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6738 | for (Inst &Instr : Node->getInsts()) { |
| 6739 | // Check whether Instr is a valid producer. |
| 6740 | Variable *Dest = Instr.getDest(); |
| 6741 | if (!Instr.isDeleted() // only consider non-deleted instructions; and |
| 6742 | && Dest // only instructions with an actual dest var; and |
| 6743 | && Dest->getType() == IceType_i1 // only bool-type dest vars; and |
| 6744 | && BoolFolding::shouldTrackProducer(Instr)) { // white-listed instr. |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6745 | KnownComputations.emplace(Dest->getIndex(), |
| 6746 | ComputationEntry(&Instr, IceType_i1)); |
| 6747 | } |
| 6748 | if (!Instr.isDeleted() // only consider non-deleted instructions; and |
| 6749 | && Dest // only instructions with an actual dest var; and |
| 6750 | && isScalarFloatingType(Dest->getType()) // fp-type only dest vars; and |
| 6751 | && FpFolding::shouldTrackProducer(Instr)) { // white-listed instr. |
| 6752 | KnownComputations.emplace(Dest->getIndex(), |
| 6753 | ComputationEntry(&Instr, Dest->getType())); |
| 6754 | } |
| 6755 | if (!Instr.isDeleted() // only consider non-deleted instructions; and |
| 6756 | && Dest // only instructions with an actual dest var; and |
| 6757 | && Dest->getType() == IceType_i32 // i32 only dest vars; and |
| 6758 | && IntFolding::shouldTrackProducer(Instr)) { // white-listed instr. |
| 6759 | KnownComputations.emplace(Dest->getIndex(), |
| 6760 | ComputationEntry(&Instr, IceType_i32)); |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6761 | } |
| 6762 | // Check each src variable against the map. |
| 6763 | FOREACH_VAR_IN_INST(Var, Instr) { |
| 6764 | SizeT VarNum = Var->getIndex(); |
| 6765 | auto ComputationIter = KnownComputations.find(VarNum); |
| 6766 | if (ComputationIter == KnownComputations.end()) { |
| 6767 | continue; |
| 6768 | } |
| 6769 | |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6770 | ++ComputationIter->second.NumUses; |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6771 | switch (ComputationIter->second.ComputationType) { |
| 6772 | default: |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6773 | KnownComputations.erase(VarNum); |
| 6774 | continue; |
John Porto | eb13acc | 2015-12-09 05:10:58 -0800 | [diff] [blame] | 6775 | case IceType_i1: |
| 6776 | if (!BoolFolding::isValidConsumer(Instr)) { |
| 6777 | KnownComputations.erase(VarNum); |
| 6778 | continue; |
| 6779 | } |
| 6780 | break; |
| 6781 | case IceType_i32: |
| 6782 | if (IndexOfVarInInst(Var) != 1 || !IntFolding::isValidConsumer(Instr)) { |
| 6783 | KnownComputations.erase(VarNum); |
| 6784 | continue; |
| 6785 | } |
| 6786 | break; |
| 6787 | case IceType_f32: |
| 6788 | case IceType_f64: |
| 6789 | if (IndexOfVarInInst(Var) != 1 || !FpFolding::isValidConsumer(Instr)) { |
| 6790 | KnownComputations.erase(VarNum); |
| 6791 | continue; |
| 6792 | } |
| 6793 | break; |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6794 | } |
| 6795 | |
| 6796 | if (Instr.isLastUse(Var)) { |
| 6797 | ComputationIter->second.IsLiveOut = false; |
| 6798 | } |
| 6799 | } |
| 6800 | } |
| 6801 | |
| 6802 | for (auto Iter = KnownComputations.begin(), End = KnownComputations.end(); |
| 6803 | Iter != End;) { |
| 6804 | // Disable the folding if its dest may be live beyond this block. |
John Porto | 7b3d9cb | 2015-11-11 14:26:57 -0800 | [diff] [blame] | 6805 | if (Iter->second.IsLiveOut || Iter->second.NumUses > 1) { |
John Porto | 4a5e6d0 | 2015-11-04 09:32:55 -0800 | [diff] [blame] | 6806 | Iter = KnownComputations.erase(Iter); |
| 6807 | continue; |
| 6808 | } |
| 6809 | |
| 6810 | // Mark as "dead" rather than outright deleting. This is so that other |
| 6811 | // peephole style optimizations during or before lowering have access to |
| 6812 | // this instruction in undeleted form. See for example |
| 6813 | // tryOptimizedCmpxchgCmpBr(). |
| 6814 | Iter->second.Instr->setDead(); |
| 6815 | ++Iter; |
| 6816 | } |
| 6817 | } |
| 6818 | |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6819 | TargetARM32::Sandboxer::Sandboxer(TargetARM32 *Target, |
| 6820 | InstBundleLock::Option BundleOption) |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6821 | : Target(Target), BundleOption(BundleOption) {} |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6822 | |
John Porto | 3bf335f | 2016-01-15 11:17:55 -0800 | [diff] [blame] | 6823 | TargetARM32::Sandboxer::~Sandboxer() {} |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6824 | |
| 6825 | namespace { |
| 6826 | OperandARM32FlexImm *indirectBranchBicMask(Cfg *Func) { |
| 6827 | constexpr uint32_t Imm8 = 0xFC; // 0xC000000F |
| 6828 | constexpr uint32_t RotateAmt = 2; |
| 6829 | return OperandARM32FlexImm::create(Func, IceType_i32, Imm8, RotateAmt); |
| 6830 | } |
| 6831 | |
| 6832 | OperandARM32FlexImm *memOpBicMask(Cfg *Func) { |
| 6833 | constexpr uint32_t Imm8 = 0x0C; // 0xC0000000 |
| 6834 | constexpr uint32_t RotateAmt = 2; |
| 6835 | return OperandARM32FlexImm::create(Func, IceType_i32, Imm8, RotateAmt); |
| 6836 | } |
| 6837 | |
| 6838 | static bool baseNeedsBic(Variable *Base) { |
| 6839 | return Base->getRegNum() != RegARM32::Reg_r9 && |
| 6840 | Base->getRegNum() != RegARM32::Reg_sp; |
| 6841 | } |
| 6842 | } // end of anonymous namespace |
| 6843 | |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6844 | void TargetARM32::Sandboxer::createAutoBundle() { |
| 6845 | Bundler = makeUnique<AutoBundle>(Target, BundleOption); |
| 6846 | } |
| 6847 | |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6848 | void TargetARM32::Sandboxer::add_sp(Operand *AddAmount) { |
| 6849 | Variable *SP = Target->getPhysicalRegister(RegARM32::Reg_sp); |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6850 | if (!Target->NeedSandboxing) { |
| 6851 | Target->_add(SP, SP, AddAmount); |
| 6852 | return; |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6853 | } |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6854 | createAutoBundle(); |
| 6855 | Target->_add(SP, SP, AddAmount); |
| 6856 | Target->_bic(SP, SP, memOpBicMask(Target->Func)); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6857 | } |
| 6858 | |
| 6859 | void TargetARM32::Sandboxer::align_sp(size_t Alignment) { |
| 6860 | Variable *SP = Target->getPhysicalRegister(RegARM32::Reg_sp); |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6861 | if (!Target->NeedSandboxing) { |
| 6862 | Target->alignRegisterPow2(SP, Alignment); |
| 6863 | return; |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6864 | } |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6865 | createAutoBundle(); |
| 6866 | Target->alignRegisterPow2(SP, Alignment); |
| 6867 | Target->_bic(SP, SP, memOpBicMask(Target->Func)); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6868 | } |
| 6869 | |
| 6870 | InstARM32Call *TargetARM32::Sandboxer::bl(Variable *ReturnReg, |
| 6871 | Operand *CallTarget) { |
| 6872 | if (Target->NeedSandboxing) { |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6873 | createAutoBundle(); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6874 | if (auto *CallTargetR = llvm::dyn_cast<Variable>(CallTarget)) { |
| 6875 | Target->_bic(CallTargetR, CallTargetR, |
| 6876 | indirectBranchBicMask(Target->Func)); |
| 6877 | } |
| 6878 | } |
John Porto | 1d937a8 | 2015-12-17 06:19:34 -0800 | [diff] [blame] | 6879 | return Target->Context.insert<InstARM32Call>(ReturnReg, CallTarget); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6880 | } |
| 6881 | |
| 6882 | void TargetARM32::Sandboxer::ldr(Variable *Dest, OperandARM32Mem *Mem, |
| 6883 | CondARM32::Cond Pred) { |
| 6884 | Variable *MemBase = Mem->getBase(); |
| 6885 | if (Target->NeedSandboxing && baseNeedsBic(MemBase)) { |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6886 | createAutoBundle(); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6887 | assert(!Mem->isRegReg()); |
| 6888 | Target->_bic(MemBase, MemBase, memOpBicMask(Target->Func), Pred); |
| 6889 | } |
| 6890 | Target->_ldr(Dest, Mem, Pred); |
| 6891 | } |
| 6892 | |
| 6893 | void TargetARM32::Sandboxer::ldrex(Variable *Dest, OperandARM32Mem *Mem, |
| 6894 | CondARM32::Cond Pred) { |
| 6895 | Variable *MemBase = Mem->getBase(); |
| 6896 | if (Target->NeedSandboxing && baseNeedsBic(MemBase)) { |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6897 | createAutoBundle(); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6898 | assert(!Mem->isRegReg()); |
| 6899 | Target->_bic(MemBase, MemBase, memOpBicMask(Target->Func), Pred); |
| 6900 | } |
| 6901 | Target->_ldrex(Dest, Mem, Pred); |
| 6902 | } |
| 6903 | |
| 6904 | void TargetARM32::Sandboxer::reset_sp(Variable *Src) { |
| 6905 | Variable *SP = Target->getPhysicalRegister(RegARM32::Reg_sp); |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6906 | if (!Target->NeedSandboxing) { |
| 6907 | Target->_mov_redefined(SP, Src); |
| 6908 | return; |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6909 | } |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6910 | createAutoBundle(); |
| 6911 | Target->_mov_redefined(SP, Src); |
| 6912 | Target->_bic(SP, SP, memOpBicMask(Target->Func)); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6913 | } |
| 6914 | |
| 6915 | void TargetARM32::Sandboxer::ret(Variable *RetAddr, Variable *RetValue) { |
| 6916 | if (Target->NeedSandboxing) { |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6917 | createAutoBundle(); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6918 | Target->_bic(RetAddr, RetAddr, indirectBranchBicMask(Target->Func)); |
| 6919 | } |
| 6920 | Target->_ret(RetAddr, RetValue); |
| 6921 | } |
| 6922 | |
| 6923 | void TargetARM32::Sandboxer::str(Variable *Src, OperandARM32Mem *Mem, |
| 6924 | CondARM32::Cond Pred) { |
| 6925 | Variable *MemBase = Mem->getBase(); |
| 6926 | if (Target->NeedSandboxing && baseNeedsBic(MemBase)) { |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6927 | createAutoBundle(); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6928 | assert(!Mem->isRegReg()); |
| 6929 | Target->_bic(MemBase, MemBase, memOpBicMask(Target->Func), Pred); |
| 6930 | } |
| 6931 | Target->_str(Src, Mem, Pred); |
| 6932 | } |
| 6933 | |
| 6934 | void TargetARM32::Sandboxer::strex(Variable *Dest, Variable *Src, |
| 6935 | OperandARM32Mem *Mem, CondARM32::Cond Pred) { |
| 6936 | Variable *MemBase = Mem->getBase(); |
| 6937 | if (Target->NeedSandboxing && baseNeedsBic(MemBase)) { |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6938 | createAutoBundle(); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6939 | assert(!Mem->isRegReg()); |
| 6940 | Target->_bic(MemBase, MemBase, memOpBicMask(Target->Func), Pred); |
| 6941 | } |
| 6942 | Target->_strex(Dest, Src, Mem, Pred); |
| 6943 | } |
| 6944 | |
| 6945 | void TargetARM32::Sandboxer::sub_sp(Operand *SubAmount) { |
| 6946 | Variable *SP = Target->getPhysicalRegister(RegARM32::Reg_sp); |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6947 | if (!Target->NeedSandboxing) { |
| 6948 | Target->_sub(SP, SP, SubAmount); |
| 6949 | return; |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6950 | } |
John Porto | a1cdd57 | 2016-03-01 15:19:29 -0800 | [diff] [blame] | 6951 | createAutoBundle(); |
| 6952 | Target->_sub(SP, SP, SubAmount); |
| 6953 | Target->_bic(SP, SP, memOpBicMask(Target->Func)); |
John Porto | 52b5157 | 2015-12-05 14:16:25 -0800 | [diff] [blame] | 6954 | } |
| 6955 | |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6956 | TargetDataARM32::TargetDataARM32(GlobalContext *Ctx) |
| 6957 | : TargetDataLowering(Ctx) {} |
| 6958 | |
John Porto | 8b1a705 | 2015-06-17 13:20:08 -0700 | [diff] [blame] | 6959 | void TargetDataARM32::lowerGlobals(const VariableDeclarationList &Vars, |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 6960 | const std::string &SectionSuffix) { |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 6961 | const bool IsPIC = getFlags().getUseNonsfi(); |
| 6962 | switch (getFlags().getOutFileType()) { |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6963 | case FT_Elf: { |
| 6964 | ELFObjectWriter *Writer = Ctx->getObjectWriter(); |
Jim Stichnoth | 8ff4b28 | 2016-01-04 15:39:06 -0800 | [diff] [blame] | 6965 | Writer->writeDataSection(Vars, llvm::ELF::R_ARM_ABS32, SectionSuffix, |
| 6966 | IsPIC); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6967 | } break; |
| 6968 | case FT_Asm: |
| 6969 | case FT_Iasm: { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 6970 | OstreamLocker _(Ctx); |
John Porto | 8b1a705 | 2015-06-17 13:20:08 -0700 | [diff] [blame] | 6971 | for (const VariableDeclaration *Var : Vars) { |
Jim Stichnoth | dd6dcfa | 2016-04-18 12:52:09 -0700 | [diff] [blame] | 6972 | if (getFlags().matchTranslateOnly(Var->getName(), 0)) { |
John Porto | 8b1a705 | 2015-06-17 13:20:08 -0700 | [diff] [blame] | 6973 | emitGlobal(*Var, SectionSuffix); |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 6974 | } |
| 6975 | } |
| 6976 | } break; |
| 6977 | } |
| 6978 | } |
| 6979 | |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 6980 | namespace { |
| 6981 | template <typename T> struct ConstantPoolEmitterTraits; |
| 6982 | |
| 6983 | static_assert(sizeof(uint64_t) == 8, |
| 6984 | "uint64_t is supposed to be 8 bytes wide."); |
| 6985 | |
| 6986 | // TODO(jpp): implement the following when implementing constant randomization: |
| 6987 | // * template <> struct ConstantPoolEmitterTraits<uint8_t> |
| 6988 | // * template <> struct ConstantPoolEmitterTraits<uint16_t> |
| 6989 | // * template <> struct ConstantPoolEmitterTraits<uint32_t> |
| 6990 | template <> struct ConstantPoolEmitterTraits<float> { |
| 6991 | using ConstantType = ConstantFloat; |
| 6992 | static constexpr Type IceType = IceType_f32; |
| 6993 | // AsmTag and TypeName can't be constexpr because llvm::StringRef is unhappy |
| 6994 | // about them being constexpr. |
| 6995 | static const char AsmTag[]; |
| 6996 | static const char TypeName[]; |
| 6997 | static uint64_t bitcastToUint64(float Value) { |
| 6998 | static_assert(sizeof(Value) == sizeof(uint32_t), |
| 6999 | "Float should be 4 bytes."); |
Jim Stichnoth | b0051df | 2016-01-13 11:39:15 -0800 | [diff] [blame] | 7000 | const uint32_t IntValue = Utils::bitCopy<uint32_t>(Value); |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7001 | return static_cast<uint64_t>(IntValue); |
| 7002 | } |
| 7003 | }; |
| 7004 | const char ConstantPoolEmitterTraits<float>::AsmTag[] = ".long"; |
| 7005 | const char ConstantPoolEmitterTraits<float>::TypeName[] = "f32"; |
| 7006 | |
| 7007 | template <> struct ConstantPoolEmitterTraits<double> { |
| 7008 | using ConstantType = ConstantDouble; |
| 7009 | static constexpr Type IceType = IceType_f64; |
| 7010 | static const char AsmTag[]; |
| 7011 | static const char TypeName[]; |
| 7012 | static uint64_t bitcastToUint64(double Value) { |
| 7013 | static_assert(sizeof(double) == sizeof(uint64_t), |
| 7014 | "Double should be 8 bytes."); |
Jim Stichnoth | b0051df | 2016-01-13 11:39:15 -0800 | [diff] [blame] | 7015 | return Utils::bitCopy<uint64_t>(Value); |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7016 | } |
| 7017 | }; |
| 7018 | const char ConstantPoolEmitterTraits<double>::AsmTag[] = ".quad"; |
| 7019 | const char ConstantPoolEmitterTraits<double>::TypeName[] = "f64"; |
| 7020 | |
| 7021 | template <typename T> |
| 7022 | void emitConstant( |
Jim Stichnoth | 98ba006 | 2016-03-07 09:26:22 -0800 | [diff] [blame] | 7023 | Ostream &Str, |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7024 | const typename ConstantPoolEmitterTraits<T>::ConstantType *Const) { |
| 7025 | using Traits = ConstantPoolEmitterTraits<T>; |
Jim Stichnoth | 467ffe5 | 2016-03-29 15:01:06 -0700 | [diff] [blame] | 7026 | Str << Const->getLabelName(); |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7027 | Str << ":\n\t" << Traits::AsmTag << "\t0x"; |
| 7028 | T Value = Const->getValue(); |
| 7029 | Str.write_hex(Traits::bitcastToUint64(Value)); |
Jim Stichnoth | 751e27e | 2015-12-16 12:40:31 -0800 | [diff] [blame] | 7030 | Str << "\t/* " << Traits::TypeName << " " << Value << " */\n"; |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7031 | } |
| 7032 | |
| 7033 | template <typename T> void emitConstantPool(GlobalContext *Ctx) { |
| 7034 | if (!BuildDefs::dump()) { |
| 7035 | return; |
| 7036 | } |
| 7037 | |
| 7038 | using Traits = ConstantPoolEmitterTraits<T>; |
| 7039 | static constexpr size_t MinimumAlignment = 4; |
| 7040 | SizeT Align = std::max(MinimumAlignment, typeAlignInBytes(Traits::IceType)); |
| 7041 | assert((Align % 4) == 0 && "Constants should be aligned"); |
| 7042 | Ostream &Str = Ctx->getStrEmit(); |
| 7043 | ConstantList Pool = Ctx->getConstantPool(Traits::IceType); |
| 7044 | |
| 7045 | Str << "\t.section\t.rodata.cst" << Align << ",\"aM\",%progbits," << Align |
| 7046 | << "\n" |
| 7047 | << "\t.align\t" << Align << "\n"; |
| 7048 | |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 7049 | if (getFlags().getReorderPooledConstants()) { |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7050 | // TODO(jpp): add constant pooling. |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 7051 | UnimplementedError(getFlags()); |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7052 | } |
| 7053 | |
| 7054 | for (Constant *C : Pool) { |
| 7055 | if (!C->getShouldBePooled()) { |
| 7056 | continue; |
| 7057 | } |
| 7058 | |
Jim Stichnoth | 98ba006 | 2016-03-07 09:26:22 -0800 | [diff] [blame] | 7059 | emitConstant<T>(Str, llvm::dyn_cast<typename Traits::ConstantType>(C)); |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7060 | } |
| 7061 | } |
| 7062 | } // end of anonymous namespace |
| 7063 | |
John Porto | 0f86d03 | 2015-06-15 07:44:27 -0700 | [diff] [blame] | 7064 | void TargetDataARM32::lowerConstants() { |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 7065 | if (getFlags().getDisableTranslation()) |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 7066 | return; |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 7067 | switch (getFlags().getOutFileType()) { |
John Porto | 7d59513 | 2016-02-01 12:43:13 -0800 | [diff] [blame] | 7068 | case FT_Elf: { |
| 7069 | ELFObjectWriter *Writer = Ctx->getObjectWriter(); |
| 7070 | Writer->writeConstantPool<ConstantFloat>(IceType_f32); |
| 7071 | Writer->writeConstantPool<ConstantDouble>(IceType_f64); |
| 7072 | } break; |
Karl Schimpf | c5abdc1 | 2015-10-09 13:29:13 -0700 | [diff] [blame] | 7073 | case FT_Asm: |
| 7074 | case FT_Iasm: { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 7075 | OstreamLocker _(Ctx); |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7076 | emitConstantPool<float>(Ctx); |
| 7077 | emitConstantPool<double>(Ctx); |
| 7078 | break; |
| 7079 | } |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7080 | } |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 7081 | } |
| 7082 | |
Andrew Scull | 86df4e9 | 2015-07-30 13:54:44 -0700 | [diff] [blame] | 7083 | void TargetDataARM32::lowerJumpTables() { |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 7084 | if (getFlags().getDisableTranslation()) |
Andrew Scull | 86df4e9 | 2015-07-30 13:54:44 -0700 | [diff] [blame] | 7085 | return; |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 7086 | switch (getFlags().getOutFileType()) { |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7087 | case FT_Elf: |
John Porto | 7d59513 | 2016-02-01 12:43:13 -0800 | [diff] [blame] | 7088 | if (!Ctx->getJumpTables().empty()) { |
| 7089 | llvm::report_fatal_error("ARM32 does not support jump tables yet."); |
| 7090 | } |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7091 | break; |
| 7092 | case FT_Asm: |
| 7093 | // Already emitted from Cfg |
| 7094 | break; |
| 7095 | case FT_Iasm: { |
Karl Schimpf | c5abdc1 | 2015-10-09 13:29:13 -0700 | [diff] [blame] | 7096 | // TODO(kschimpf): Fill this in when we get more information. |
John Porto | a83bfde | 2015-09-18 08:43:02 -0700 | [diff] [blame] | 7097 | break; |
| 7098 | } |
| 7099 | } |
Andrew Scull | 86df4e9 | 2015-07-30 13:54:44 -0700 | [diff] [blame] | 7100 | } |
| 7101 | |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7102 | TargetHeaderARM32::TargetHeaderARM32(GlobalContext *Ctx) |
Karl Schimpf | d469994 | 2016-04-02 09:55:31 -0700 | [diff] [blame] | 7103 | : TargetHeaderLowering(Ctx), CPUFeatures(getFlags()) {} |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7104 | |
| 7105 | void TargetHeaderARM32::lower() { |
John Porto | f5f02f7 | 2015-11-09 14:52:40 -0800 | [diff] [blame] | 7106 | OstreamLocker _(Ctx); |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7107 | Ostream &Str = Ctx->getStrEmit(); |
| 7108 | Str << ".syntax unified\n"; |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 7109 | // Emit build attributes in format: .eabi_attribute TAG, VALUE. See Sec. 2 of |
| 7110 | // "Addenda to, and Errata in the ABI for the ARM architecture" |
| 7111 | // http://infocenter.arm.com |
| 7112 | // /help/topic/com.arm.doc.ihi0045d/IHI0045D_ABI_addenda.pdf |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7113 | // |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 7114 | // Tag_conformance should be be emitted first in a file-scope sub-subsection |
| 7115 | // of the first public subsection of the attributes. |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7116 | Str << ".eabi_attribute 67, \"2.09\" @ Tag_conformance\n"; |
Andrew Scull | 57e1268 | 2015-09-16 11:30:19 -0700 | [diff] [blame] | 7117 | // Chromebooks are at least A15, but do A9 for higher compat. For some |
| 7118 | // reason, the LLVM ARM asm parser has the .cpu directive override the mattr |
| 7119 | // specified on the commandline. So to test hwdiv, we need to set the .cpu |
| 7120 | // directive higher (can't just rely on --mattr=...). |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 7121 | if (CPUFeatures.hasFeature(TargetARM32Features::HWDivArm)) { |
| 7122 | Str << ".cpu cortex-a15\n"; |
| 7123 | } else { |
| 7124 | Str << ".cpu cortex-a9\n"; |
| 7125 | } |
| 7126 | Str << ".eabi_attribute 6, 10 @ Tag_CPU_arch: ARMv7\n" |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7127 | << ".eabi_attribute 7, 65 @ Tag_CPU_arch_profile: App profile\n"; |
| 7128 | Str << ".eabi_attribute 8, 1 @ Tag_ARM_ISA_use: Yes\n" |
| 7129 | << ".eabi_attribute 9, 2 @ Tag_THUMB_ISA_use: Thumb-2\n"; |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7130 | Str << ".fpu neon\n" |
| 7131 | << ".eabi_attribute 17, 1 @ Tag_ABI_PCS_GOT_use: permit directly\n" |
| 7132 | << ".eabi_attribute 20, 1 @ Tag_ABI_FP_denormal\n" |
| 7133 | << ".eabi_attribute 21, 1 @ Tag_ABI_FP_exceptions\n" |
| 7134 | << ".eabi_attribute 23, 3 @ Tag_ABI_FP_number_model: IEEE 754\n" |
| 7135 | << ".eabi_attribute 34, 1 @ Tag_CPU_unaligned_access\n" |
| 7136 | << ".eabi_attribute 24, 1 @ Tag_ABI_align_needed: 8-byte\n" |
| 7137 | << ".eabi_attribute 25, 1 @ Tag_ABI_align_preserved: 8-byte\n" |
| 7138 | << ".eabi_attribute 28, 1 @ Tag_ABI_VFP_args\n" |
| 7139 | << ".eabi_attribute 36, 1 @ Tag_FP_HP_extension\n" |
| 7140 | << ".eabi_attribute 38, 1 @ Tag_ABI_FP_16bit_format\n" |
| 7141 | << ".eabi_attribute 42, 1 @ Tag_MPextension_use\n" |
| 7142 | << ".eabi_attribute 68, 1 @ Tag_Virtualization_use\n"; |
Jan Voung | 6ec369e | 2015-06-30 11:03:15 -0700 | [diff] [blame] | 7143 | if (CPUFeatures.hasFeature(TargetARM32Features::HWDivArm)) { |
| 7144 | Str << ".eabi_attribute 44, 2 @ Tag_DIV_use\n"; |
| 7145 | } |
Jan Voung | fb79284 | 2015-06-11 15:27:50 -0700 | [diff] [blame] | 7146 | // Technically R9 is used for TLS with Sandboxing, and we reserve it. |
| 7147 | // However, for compatibility with current NaCl LLVM, don't claim that. |
| 7148 | Str << ".eabi_attribute 14, 3 @ Tag_ABI_PCS_R9_use: Not used\n"; |
| 7149 | } |
| 7150 | |
John Porto | e82b560 | 2016-02-24 15:58:55 -0800 | [diff] [blame] | 7151 | SmallBitVector TargetARM32::TypeToRegisterSet[RegARM32::RCARM32_NUM]; |
| 7152 | SmallBitVector TargetARM32::TypeToRegisterSetUnfiltered[RegARM32::RCARM32_NUM]; |
| 7153 | SmallBitVector TargetARM32::RegisterAliases[RegARM32::Reg_NUM]; |
Jim Stichnoth | 94844f1 | 2015-11-04 16:06:16 -0800 | [diff] [blame] | 7154 | |
John Porto | 4a56686 | 2016-01-04 09:33:41 -0800 | [diff] [blame] | 7155 | } // end of namespace ARM32 |
Jan Voung | b36ad9b | 2015-04-21 17:01:49 -0700 | [diff] [blame] | 7156 | } // end of namespace Ice |