Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 1 | //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | /// \file |
| 11 | /// \brief Insert wait instructions for memory reads and writes. |
| 12 | /// |
| 13 | /// Memory reads and writes are issued asynchronously, so we need to insert |
| 14 | /// S_WAITCNT instructions when we want to access any of their results or |
| 15 | /// overwrite any register that's used asynchronously. |
| 16 | // |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | |
| 19 | #include "AMDGPU.h" |
Eric Christopher | d913448 | 2014-08-04 21:25:23 +0000 | [diff] [blame] | 20 | #include "AMDGPUSubtarget.h" |
Matt Arsenault | 9783e00 | 2014-09-29 15:50:26 +0000 | [diff] [blame] | 21 | #include "SIDefines.h" |
Matt Arsenault | 1fd0c62 | 2014-09-29 15:53:15 +0000 | [diff] [blame] | 22 | #include "SIInstrInfo.h" |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 23 | #include "SIMachineFunctionInfo.h" |
| 24 | #include "llvm/CodeGen/MachineFunction.h" |
| 25 | #include "llvm/CodeGen/MachineFunctionPass.h" |
| 26 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
| 27 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 28 | |
Tom Stellard | 6e1967e | 2016-02-05 17:42:38 +0000 | [diff] [blame] | 29 | #define DEBUG_TYPE "si-insert-waits" |
| 30 | |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 31 | using namespace llvm; |
| 32 | |
| 33 | namespace { |
| 34 | |
| 35 | /// \brief One variable for each of the hardware counters |
| 36 | typedef union { |
| 37 | struct { |
| 38 | unsigned VM; |
| 39 | unsigned EXP; |
| 40 | unsigned LGKM; |
| 41 | } Named; |
| 42 | unsigned Array[3]; |
| 43 | |
| 44 | } Counters; |
| 45 | |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 46 | typedef enum { |
| 47 | OTHER, |
| 48 | SMEM, |
| 49 | VMEM |
| 50 | } InstType; |
| 51 | |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 52 | typedef Counters RegCounters[512]; |
| 53 | typedef std::pair<unsigned, unsigned> RegInterval; |
| 54 | |
| 55 | class SIInsertWaits : public MachineFunctionPass { |
| 56 | |
| 57 | private: |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 58 | const SIInstrInfo *TII; |
Bill Wendling | 37e9adb | 2013-06-07 20:28:55 +0000 | [diff] [blame] | 59 | const SIRegisterInfo *TRI; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 60 | const MachineRegisterInfo *MRI; |
| 61 | |
| 62 | /// \brief Constant hardware limits |
| 63 | static const Counters WaitCounts; |
| 64 | |
| 65 | /// \brief Constant zero value |
| 66 | static const Counters ZeroCounts; |
| 67 | |
| 68 | /// \brief Counter values we have already waited on. |
| 69 | Counters WaitedOn; |
| 70 | |
| 71 | /// \brief Counter values for last instruction issued. |
| 72 | Counters LastIssued; |
| 73 | |
| 74 | /// \brief Registers used by async instructions. |
| 75 | RegCounters UsedRegs; |
| 76 | |
| 77 | /// \brief Registers defined by async instructions. |
| 78 | RegCounters DefinedRegs; |
| 79 | |
| 80 | /// \brief Different export instruction types seen since last wait. |
| 81 | unsigned ExpInstrTypesSeen; |
| 82 | |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 83 | /// \brief Type of the last opcode. |
| 84 | InstType LastOpcodeType; |
| 85 | |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 86 | bool LastInstWritesM0; |
| 87 | |
Marek Olsak | 8e9cc63 | 2016-01-13 17:23:09 +0000 | [diff] [blame] | 88 | /// \brief Whether the machine function returns void |
| 89 | bool ReturnsVoid; |
| 90 | |
Tom Stellard | 3096176 | 2016-02-08 19:49:20 +0000 | [diff] [blame] | 91 | /// Whether the VCCZ bit is possibly corrupt |
| 92 | bool VCCZCorrupt; |
| 93 | |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 94 | /// \brief Get increment/decrement amount for this instruction. |
| 95 | Counters getHwCounts(MachineInstr &MI); |
| 96 | |
| 97 | /// \brief Is operand relevant for async execution? |
| 98 | bool isOpRelevant(MachineOperand &Op); |
| 99 | |
| 100 | /// \brief Get register interval an operand affects. |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 101 | RegInterval getRegInterval(const TargetRegisterClass *RC, |
| 102 | const MachineOperand &Reg) const; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 103 | |
| 104 | /// \brief Handle instructions async components |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 105 | void pushInstruction(MachineBasicBlock &MBB, |
| 106 | MachineBasicBlock::iterator I); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 107 | |
| 108 | /// \brief Insert the actual wait instruction |
| 109 | bool insertWait(MachineBasicBlock &MBB, |
| 110 | MachineBasicBlock::iterator I, |
| 111 | const Counters &Counts); |
| 112 | |
Christian Konig | 862fd9f | 2013-03-01 09:46:04 +0000 | [diff] [blame] | 113 | /// \brief Do we need def2def checks? |
| 114 | bool unorderedDefines(MachineInstr &MI); |
| 115 | |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 116 | /// \brief Resolve all operand dependencies to counter requirements |
| 117 | Counters handleOperands(MachineInstr &MI); |
| 118 | |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 119 | /// \brief Insert S_NOP between an instruction writing M0 and S_SENDMSG. |
| 120 | void handleSendMsg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I); |
| 121 | |
Tom Stellard | 331f981 | 2016-03-14 17:05:56 +0000 | [diff] [blame] | 122 | /// \param DPP The DPP instruction |
| 123 | /// \param SearchI The iterator to start look for hazards. |
| 124 | /// \param SearchMBB The basic block we are operating on. |
| 125 | /// \param WaitStates Then number of wait states that need to be inserted |
| 126 | /// When a hazard is detected. |
| 127 | void insertDPPWaitStates(MachineBasicBlock::iterator DPP, |
| 128 | MachineBasicBlock::reverse_iterator SearchI, |
| 129 | MachineBasicBlock *SearchMBB, |
| 130 | unsigned WaitStates); |
| 131 | |
| 132 | void insertDPPWaitStates(MachineBasicBlock::iterator DPP); |
| 133 | |
Tom Stellard | 3096176 | 2016-02-08 19:49:20 +0000 | [diff] [blame] | 134 | /// Return true if there are LGKM instrucitons that haven't been waited on |
| 135 | /// yet. |
| 136 | bool hasOutstandingLGKM() const; |
| 137 | |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 138 | public: |
Tom Stellard | 6e1967e | 2016-02-05 17:42:38 +0000 | [diff] [blame] | 139 | static char ID; |
| 140 | |
| 141 | SIInsertWaits() : |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 142 | MachineFunctionPass(ID), |
Craig Topper | 062a2ba | 2014-04-25 05:30:21 +0000 | [diff] [blame] | 143 | TII(nullptr), |
| 144 | TRI(nullptr), |
Tom Stellard | 3096176 | 2016-02-08 19:49:20 +0000 | [diff] [blame] | 145 | ExpInstrTypesSeen(0), |
| 146 | VCCZCorrupt(false) { } |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 147 | |
Craig Topper | 5656db4 | 2014-04-29 07:57:24 +0000 | [diff] [blame] | 148 | bool runOnMachineFunction(MachineFunction &MF) override; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 149 | |
Craig Topper | 5656db4 | 2014-04-29 07:57:24 +0000 | [diff] [blame] | 150 | const char *getPassName() const override { |
Matt Arsenault | 0cb8517 | 2015-09-25 17:21:28 +0000 | [diff] [blame] | 151 | return "SI insert wait instructions"; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 152 | } |
| 153 | |
Matt Arsenault | 0cb8517 | 2015-09-25 17:21:28 +0000 | [diff] [blame] | 154 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 155 | AU.setPreservesCFG(); |
| 156 | MachineFunctionPass::getAnalysisUsage(AU); |
| 157 | } |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 158 | }; |
| 159 | |
| 160 | } // End anonymous namespace |
| 161 | |
Tom Stellard | 6e1967e | 2016-02-05 17:42:38 +0000 | [diff] [blame] | 162 | INITIALIZE_PASS_BEGIN(SIInsertWaits, DEBUG_TYPE, |
| 163 | "SI Insert Waits", false, false) |
| 164 | INITIALIZE_PASS_END(SIInsertWaits, DEBUG_TYPE, |
| 165 | "SI Insert Waits", false, false) |
| 166 | |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 167 | char SIInsertWaits::ID = 0; |
| 168 | |
Tom Stellard | 6e1967e | 2016-02-05 17:42:38 +0000 | [diff] [blame] | 169 | char &llvm::SIInsertWaitsID = SIInsertWaits::ID; |
| 170 | |
| 171 | FunctionPass *llvm::createSIInsertWaitsPass() { |
| 172 | return new SIInsertWaits(); |
| 173 | } |
| 174 | |
Tom Stellard | 3d2c852 | 2016-01-28 17:13:44 +0000 | [diff] [blame] | 175 | const Counters SIInsertWaits::WaitCounts = { { 15, 7, 15 } }; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 176 | const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } }; |
| 177 | |
Tom Stellard | 3096176 | 2016-02-08 19:49:20 +0000 | [diff] [blame] | 178 | static bool readsVCCZ(unsigned Opcode) { |
Matt Arsenault | f2dcb47 | 2016-03-02 04:12:39 +0000 | [diff] [blame] | 179 | return Opcode == AMDGPU::S_CBRANCH_VCCNZ || Opcode == AMDGPU::S_CBRANCH_VCCZ; |
Tom Stellard | 3096176 | 2016-02-08 19:49:20 +0000 | [diff] [blame] | 180 | } |
| 181 | |
| 182 | bool SIInsertWaits::hasOutstandingLGKM() const { |
| 183 | return WaitedOn.Named.LGKM != LastIssued.Named.LGKM; |
| 184 | } |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 185 | |
| 186 | Counters SIInsertWaits::getHwCounts(MachineInstr &MI) { |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 187 | uint64_t TSFlags = MI.getDesc().TSFlags; |
Matt Arsenault | e66621b | 2015-09-24 19:52:27 +0000 | [diff] [blame] | 188 | Counters Result = { { 0, 0, 0 } }; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 189 | |
| 190 | Result.Named.VM = !!(TSFlags & SIInstrFlags::VM_CNT); |
| 191 | |
| 192 | // Only consider stores or EXP for EXP_CNT |
| 193 | Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT && |
Christian Konig | 862fd9f | 2013-03-01 09:46:04 +0000 | [diff] [blame] | 194 | (MI.getOpcode() == AMDGPU::EXP || MI.getDesc().mayStore())); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 195 | |
| 196 | // LGKM may uses larger values |
| 197 | if (TSFlags & SIInstrFlags::LGKM_CNT) { |
| 198 | |
Matt Arsenault | 3add643 | 2015-10-20 04:35:43 +0000 | [diff] [blame] | 199 | if (TII->isSMRD(MI)) { |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 200 | |
Matt Arsenault | e66621b | 2015-09-24 19:52:27 +0000 | [diff] [blame] | 201 | if (MI.getNumOperands() != 0) { |
Matt Arsenault | b733f00 | 2015-10-01 22:40:35 +0000 | [diff] [blame] | 202 | assert(MI.getOperand(0).isReg() && |
| 203 | "First LGKM operand must be a register!"); |
Michel Danzer | 20680b1 | 2013-08-16 16:19:24 +0000 | [diff] [blame] | 204 | |
Matt Arsenault | e66621b | 2015-09-24 19:52:27 +0000 | [diff] [blame] | 205 | // XXX - What if this is a write into a super register? |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 206 | const TargetRegisterClass *RC = TII->getOpRegClass(MI, 0); |
| 207 | unsigned Size = RC->getSize(); |
Matt Arsenault | e66621b | 2015-09-24 19:52:27 +0000 | [diff] [blame] | 208 | Result.Named.LGKM = Size > 4 ? 2 : 1; |
| 209 | } else { |
| 210 | // s_dcache_inv etc. do not have a a destination register. Assume we |
| 211 | // want a wait on these. |
| 212 | // XXX - What is the right value? |
| 213 | Result.Named.LGKM = 1; |
| 214 | } |
Michel Danzer | 20680b1 | 2013-08-16 16:19:24 +0000 | [diff] [blame] | 215 | } else { |
| 216 | // DS |
| 217 | Result.Named.LGKM = 1; |
| 218 | } |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 219 | |
| 220 | } else { |
| 221 | Result.Named.LGKM = 0; |
| 222 | } |
| 223 | |
| 224 | return Result; |
| 225 | } |
| 226 | |
| 227 | bool SIInsertWaits::isOpRelevant(MachineOperand &Op) { |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 228 | // Constants are always irrelevant |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 229 | if (!Op.isReg() || !TRI->isInAllocatableClass(Op.getReg())) |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 230 | return false; |
| 231 | |
| 232 | // Defines are always relevant |
| 233 | if (Op.isDef()) |
| 234 | return true; |
| 235 | |
| 236 | // For exports all registers are relevant |
| 237 | MachineInstr &MI = *Op.getParent(); |
| 238 | if (MI.getOpcode() == AMDGPU::EXP) |
| 239 | return true; |
| 240 | |
| 241 | // For stores the stored value is also relevant |
| 242 | if (!MI.getDesc().mayStore()) |
| 243 | return false; |
| 244 | |
Tom Stellard | b3931b8 | 2015-01-06 19:52:04 +0000 | [diff] [blame] | 245 | // Check if this operand is the value being stored. |
Tom Stellard | 2d26fe7 | 2016-02-19 15:33:13 +0000 | [diff] [blame] | 246 | // Special case for DS/FLAT instructions, since the address |
Tom Stellard | b3931b8 | 2015-01-06 19:52:04 +0000 | [diff] [blame] | 247 | // operand comes before the value operand and it may have |
| 248 | // multiple data operands. |
| 249 | |
Tom Stellard | 2d26fe7 | 2016-02-19 15:33:13 +0000 | [diff] [blame] | 250 | if (TII->isDS(MI) || TII->isFLAT(MI)) { |
Tom Stellard | b3931b8 | 2015-01-06 19:52:04 +0000 | [diff] [blame] | 251 | MachineOperand *Data = TII->getNamedOperand(MI, AMDGPU::OpName::data); |
| 252 | if (Data && Op.isIdenticalTo(*Data)) |
| 253 | return true; |
Tom Stellard | 2d26fe7 | 2016-02-19 15:33:13 +0000 | [diff] [blame] | 254 | } |
Tom Stellard | b3931b8 | 2015-01-06 19:52:04 +0000 | [diff] [blame] | 255 | |
Tom Stellard | 2d26fe7 | 2016-02-19 15:33:13 +0000 | [diff] [blame] | 256 | if (TII->isDS(MI)) { |
Tom Stellard | b3931b8 | 2015-01-06 19:52:04 +0000 | [diff] [blame] | 257 | MachineOperand *Data0 = TII->getNamedOperand(MI, AMDGPU::OpName::data0); |
| 258 | if (Data0 && Op.isIdenticalTo(*Data0)) |
| 259 | return true; |
| 260 | |
| 261 | MachineOperand *Data1 = TII->getNamedOperand(MI, AMDGPU::OpName::data1); |
Matt Arsenault | 8226fc4 | 2016-03-02 23:00:21 +0000 | [diff] [blame] | 262 | return Data1 && Op.isIdenticalTo(*Data1); |
Tom Stellard | b3931b8 | 2015-01-06 19:52:04 +0000 | [diff] [blame] | 263 | } |
| 264 | |
| 265 | // NOTE: This assumes that the value operand is before the |
| 266 | // address operand, and that there is only one value operand. |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 267 | for (MachineInstr::mop_iterator I = MI.operands_begin(), |
| 268 | E = MI.operands_end(); I != E; ++I) { |
| 269 | |
| 270 | if (I->isReg() && I->isUse()) |
| 271 | return Op.isIdenticalTo(*I); |
| 272 | } |
| 273 | |
| 274 | return false; |
| 275 | } |
| 276 | |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 277 | RegInterval SIInsertWaits::getRegInterval(const TargetRegisterClass *RC, |
| 278 | const MachineOperand &Reg) const { |
| 279 | unsigned Size = RC->getSize(); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 280 | assert(Size >= 4); |
| 281 | |
| 282 | RegInterval Result; |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 283 | Result.first = TRI->getEncodingValue(Reg.getReg()); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 284 | Result.second = Result.first + Size / 4; |
| 285 | |
| 286 | return Result; |
| 287 | } |
| 288 | |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 289 | void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB, |
| 290 | MachineBasicBlock::iterator I) { |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 291 | |
| 292 | // Get the hardware counter increments and sum them up |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 293 | Counters Increment = getHwCounts(*I); |
Tom Stellard | bd8a085 | 2015-08-21 22:47:27 +0000 | [diff] [blame] | 294 | Counters Limit = ZeroCounts; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 295 | unsigned Sum = 0; |
| 296 | |
| 297 | for (unsigned i = 0; i < 3; ++i) { |
| 298 | LastIssued.Array[i] += Increment.Array[i]; |
Tom Stellard | bd8a085 | 2015-08-21 22:47:27 +0000 | [diff] [blame] | 299 | if (Increment.Array[i]) |
| 300 | Limit.Array[i] = LastIssued.Array[i]; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 301 | Sum += Increment.Array[i]; |
| 302 | } |
| 303 | |
| 304 | // If we don't increase anything then that's it |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 305 | if (Sum == 0) { |
| 306 | LastOpcodeType = OTHER; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 307 | return; |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 308 | } |
| 309 | |
Eric Christopher | 6c5b511 | 2015-03-11 18:43:21 +0000 | [diff] [blame] | 310 | if (MBB.getParent()->getSubtarget<AMDGPUSubtarget>().getGeneration() >= |
| 311 | AMDGPUSubtarget::VOLCANIC_ISLANDS) { |
Benjamin Kramer | df005cb | 2015-08-08 18:27:36 +0000 | [diff] [blame] | 312 | // Any occurrence of consecutive VMEM or SMEM instructions forms a VMEM |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 313 | // or SMEM clause, respectively. |
| 314 | // |
| 315 | // The temporary workaround is to break the clauses with S_NOP. |
| 316 | // |
| 317 | // The proper solution would be to allocate registers such that all source |
| 318 | // and destination registers don't overlap, e.g. this is illegal: |
| 319 | // r0 = load r2 |
| 320 | // r2 = load r0 |
Matt Arsenault | 3add643 | 2015-10-20 04:35:43 +0000 | [diff] [blame] | 321 | if ((LastOpcodeType == SMEM && TII->isSMRD(*I)) || |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 322 | (LastOpcodeType == VMEM && Increment.Named.VM)) { |
| 323 | // Insert a NOP to break the clause. |
| 324 | BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP)) |
| 325 | .addImm(0); |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 326 | LastInstWritesM0 = false; |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 327 | } |
| 328 | |
Matt Arsenault | 3add643 | 2015-10-20 04:35:43 +0000 | [diff] [blame] | 329 | if (TII->isSMRD(*I)) |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 330 | LastOpcodeType = SMEM; |
| 331 | else if (Increment.Named.VM) |
| 332 | LastOpcodeType = VMEM; |
| 333 | } |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 334 | |
| 335 | // Remember which export instructions we have seen |
| 336 | if (Increment.Named.EXP) { |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 337 | ExpInstrTypesSeen |= I->getOpcode() == AMDGPU::EXP ? 1 : 2; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 338 | } |
| 339 | |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 340 | for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) { |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 341 | MachineOperand &Op = I->getOperand(i); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 342 | if (!isOpRelevant(Op)) |
| 343 | continue; |
| 344 | |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 345 | const TargetRegisterClass *RC = TII->getOpRegClass(*I, i); |
| 346 | RegInterval Interval = getRegInterval(RC, Op); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 347 | for (unsigned j = Interval.first; j < Interval.second; ++j) { |
| 348 | |
| 349 | // Remember which registers we define |
| 350 | if (Op.isDef()) |
Tom Stellard | bd8a085 | 2015-08-21 22:47:27 +0000 | [diff] [blame] | 351 | DefinedRegs[j] = Limit; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 352 | |
| 353 | // and which one we are using |
| 354 | if (Op.isUse()) |
Tom Stellard | bd8a085 | 2015-08-21 22:47:27 +0000 | [diff] [blame] | 355 | UsedRegs[j] = Limit; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 356 | } |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | bool SIInsertWaits::insertWait(MachineBasicBlock &MBB, |
| 361 | MachineBasicBlock::iterator I, |
| 362 | const Counters &Required) { |
| 363 | |
| 364 | // End of program? No need to wait on anything |
Marek Olsak | 8e9cc63 | 2016-01-13 17:23:09 +0000 | [diff] [blame] | 365 | // A function not returning void needs to wait, because other bytecode will |
| 366 | // be appended after it and we don't know what it will be. |
| 367 | if (I != MBB.end() && I->getOpcode() == AMDGPU::S_ENDPGM && ReturnsVoid) |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 368 | return false; |
| 369 | |
| 370 | // Figure out if the async instructions execute in order |
| 371 | bool Ordered[3]; |
| 372 | |
| 373 | // VM_CNT is always ordered |
| 374 | Ordered[0] = true; |
| 375 | |
| 376 | // EXP_CNT is unordered if we have both EXP & VM-writes |
| 377 | Ordered[1] = ExpInstrTypesSeen == 3; |
| 378 | |
| 379 | // LGKM_CNT is handled as always unordered. TODO: Handle LDS and GDS |
| 380 | Ordered[2] = false; |
| 381 | |
| 382 | // The values we are going to put into the S_WAITCNT instruction |
| 383 | Counters Counts = WaitCounts; |
| 384 | |
| 385 | // Do we really need to wait? |
| 386 | bool NeedWait = false; |
| 387 | |
| 388 | for (unsigned i = 0; i < 3; ++i) { |
| 389 | |
| 390 | if (Required.Array[i] <= WaitedOn.Array[i]) |
| 391 | continue; |
| 392 | |
| 393 | NeedWait = true; |
Matt Arsenault | 9748369 | 2014-07-17 17:50:22 +0000 | [diff] [blame] | 394 | |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 395 | if (Ordered[i]) { |
| 396 | unsigned Value = LastIssued.Array[i] - Required.Array[i]; |
| 397 | |
Matt Arsenault | 9748369 | 2014-07-17 17:50:22 +0000 | [diff] [blame] | 398 | // Adjust the value to the real hardware possibilities. |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 399 | Counts.Array[i] = std::min(Value, WaitCounts.Array[i]); |
| 400 | |
| 401 | } else |
| 402 | Counts.Array[i] = 0; |
| 403 | |
Matt Arsenault | 9748369 | 2014-07-17 17:50:22 +0000 | [diff] [blame] | 404 | // Remember on what we have waited on. |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 405 | WaitedOn.Array[i] = LastIssued.Array[i] - Counts.Array[i]; |
| 406 | } |
| 407 | |
| 408 | if (!NeedWait) |
| 409 | return false; |
| 410 | |
| 411 | // Reset EXP_CNT instruction types |
| 412 | if (Counts.Named.EXP == 0) |
| 413 | ExpInstrTypesSeen = 0; |
| 414 | |
| 415 | // Build the wait instruction |
| 416 | BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)) |
| 417 | .addImm((Counts.Named.VM & 0xF) | |
| 418 | ((Counts.Named.EXP & 0x7) << 4) | |
Tom Stellard | 3d2c852 | 2016-01-28 17:13:44 +0000 | [diff] [blame] | 419 | ((Counts.Named.LGKM & 0xF) << 8)); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 420 | |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 421 | LastOpcodeType = OTHER; |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 422 | LastInstWritesM0 = false; |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 423 | return true; |
| 424 | } |
| 425 | |
| 426 | /// \brief helper function for handleOperands |
| 427 | static void increaseCounters(Counters &Dst, const Counters &Src) { |
| 428 | |
| 429 | for (unsigned i = 0; i < 3; ++i) |
| 430 | Dst.Array[i] = std::max(Dst.Array[i], Src.Array[i]); |
| 431 | } |
| 432 | |
| 433 | Counters SIInsertWaits::handleOperands(MachineInstr &MI) { |
| 434 | |
| 435 | Counters Result = ZeroCounts; |
| 436 | |
Michel Danzer | 6064f57 | 2014-01-27 07:20:44 +0000 | [diff] [blame] | 437 | // S_SENDMSG implicitly waits for all outstanding LGKM transfers to finish, |
| 438 | // but we also want to wait for any other outstanding transfers before |
| 439 | // signalling other hardware blocks |
| 440 | if (MI.getOpcode() == AMDGPU::S_SENDMSG) |
| 441 | return LastIssued; |
| 442 | |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 443 | // For each register affected by this instruction increase the result |
| 444 | // sequence. |
| 445 | // |
| 446 | // TODO: We could probably just look at explicit operands if we removed VCC / |
| 447 | // EXEC from SMRD dest reg classes. |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 448 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 449 | MachineOperand &Op = MI.getOperand(i); |
Matt Arsenault | d1d499a | 2015-10-01 21:43:15 +0000 | [diff] [blame] | 450 | if (!Op.isReg() || !TRI->isInAllocatableClass(Op.getReg())) |
| 451 | continue; |
| 452 | |
| 453 | const TargetRegisterClass *RC = TII->getOpRegClass(MI, i); |
| 454 | RegInterval Interval = getRegInterval(RC, Op); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 455 | for (unsigned j = Interval.first; j < Interval.second; ++j) { |
| 456 | |
Christian Konig | 862fd9f | 2013-03-01 09:46:04 +0000 | [diff] [blame] | 457 | if (Op.isDef()) { |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 458 | increaseCounters(Result, UsedRegs[j]); |
Christian Konig | f1fd5fa | 2013-03-18 11:33:45 +0000 | [diff] [blame] | 459 | increaseCounters(Result, DefinedRegs[j]); |
Christian Konig | 862fd9f | 2013-03-01 09:46:04 +0000 | [diff] [blame] | 460 | } |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 461 | |
| 462 | if (Op.isUse()) |
| 463 | increaseCounters(Result, DefinedRegs[j]); |
| 464 | } |
| 465 | } |
| 466 | |
| 467 | return Result; |
| 468 | } |
| 469 | |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 470 | void SIInsertWaits::handleSendMsg(MachineBasicBlock &MBB, |
| 471 | MachineBasicBlock::iterator I) { |
Eric Christopher | 6c5b511 | 2015-03-11 18:43:21 +0000 | [diff] [blame] | 472 | if (MBB.getParent()->getSubtarget<AMDGPUSubtarget>().getGeneration() < |
| 473 | AMDGPUSubtarget::VOLCANIC_ISLANDS) |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 474 | return; |
| 475 | |
| 476 | // There must be "S_NOP 0" between an instruction writing M0 and S_SENDMSG. |
| 477 | if (LastInstWritesM0 && I->getOpcode() == AMDGPU::S_SENDMSG) { |
| 478 | BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP)).addImm(0); |
| 479 | LastInstWritesM0 = false; |
| 480 | return; |
| 481 | } |
| 482 | |
| 483 | // Set whether this instruction sets M0 |
| 484 | LastInstWritesM0 = false; |
| 485 | |
| 486 | unsigned NumOperands = I->getNumOperands(); |
| 487 | for (unsigned i = 0; i < NumOperands; i++) { |
| 488 | const MachineOperand &Op = I->getOperand(i); |
| 489 | |
| 490 | if (Op.isReg() && Op.isDef() && Op.getReg() == AMDGPU::M0) |
| 491 | LastInstWritesM0 = true; |
| 492 | } |
| 493 | } |
| 494 | |
Tom Stellard | 331f981 | 2016-03-14 17:05:56 +0000 | [diff] [blame] | 495 | void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP, |
| 496 | MachineBasicBlock::reverse_iterator SearchI, |
| 497 | MachineBasicBlock *SearchMBB, |
| 498 | unsigned WaitStates) { |
| 499 | |
| 500 | MachineBasicBlock::reverse_iterator E = SearchMBB->rend(); |
| 501 | |
| 502 | for (; WaitStates > 0; --WaitStates, ++SearchI) { |
| 503 | |
| 504 | // If we have reached the start of the block, we need to check predecessors. |
| 505 | if (SearchI == E) { |
| 506 | for (MachineBasicBlock *Pred : SearchMBB->predecessors()) { |
| 507 | // We only need to check fall-through blocks. Branch instructions |
| 508 | // give us enough wait states. |
| 509 | if (Pred->getFirstTerminator() == Pred->end()) { |
| 510 | insertDPPWaitStates(DPP, Pred->rbegin(), Pred, WaitStates); |
| 511 | break; |
| 512 | } |
| 513 | } |
| 514 | return; |
| 515 | } |
| 516 | |
| 517 | for (MachineOperand &Op : SearchI->operands()) { |
| 518 | if (!Op.isReg() || !Op.isDef()) |
| 519 | continue; |
| 520 | |
| 521 | if (DPP->readsRegister(Op.getReg(), TRI)) { |
| 522 | TII->insertWaitStates(DPP, WaitStates); |
| 523 | return; |
| 524 | } |
| 525 | } |
| 526 | } |
| 527 | } |
| 528 | |
| 529 | void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP) { |
| 530 | MachineBasicBlock::reverse_iterator I(DPP); |
| 531 | insertDPPWaitStates(DPP, I, DPP->getParent(), 2); |
| 532 | } |
| 533 | |
Matt Arsenault | a0050b0 | 2014-06-19 01:19:19 +0000 | [diff] [blame] | 534 | // FIXME: Insert waits listed in Table 4.2 "Required User-Inserted Wait States" |
| 535 | // around other non-memory instructions. |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 536 | bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) { |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 537 | bool Changes = false; |
| 538 | |
Eric Christopher | fc6de42 | 2014-08-05 02:39:49 +0000 | [diff] [blame] | 539 | TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo()); |
| 540 | TRI = |
| 541 | static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo()); |
Bill Wendling | 37e9adb | 2013-06-07 20:28:55 +0000 | [diff] [blame] | 542 | |
Tom Stellard | 3096176 | 2016-02-08 19:49:20 +0000 | [diff] [blame] | 543 | const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>(); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 544 | MRI = &MF.getRegInfo(); |
| 545 | |
| 546 | WaitedOn = ZeroCounts; |
| 547 | LastIssued = ZeroCounts; |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 548 | LastOpcodeType = OTHER; |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 549 | LastInstWritesM0 = false; |
Marek Olsak | 8e9cc63 | 2016-01-13 17:23:09 +0000 | [diff] [blame] | 550 | ReturnsVoid = MF.getInfo<SIMachineFunctionInfo>()->returnsVoid(); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 551 | |
| 552 | memset(&UsedRegs, 0, sizeof(UsedRegs)); |
| 553 | memset(&DefinedRegs, 0, sizeof(DefinedRegs)); |
| 554 | |
| 555 | for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); |
| 556 | BI != BE; ++BI) { |
| 557 | |
| 558 | MachineBasicBlock &MBB = *BI; |
| 559 | for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); |
| 560 | I != E; ++I) { |
| 561 | |
Tom Stellard | 3096176 | 2016-02-08 19:49:20 +0000 | [diff] [blame] | 562 | if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) { |
| 563 | // There is a hardware bug on CI/SI where SMRD instruction may corrupt |
| 564 | // vccz bit, so when we detect that an instruction may read from a |
| 565 | // corrupt vccz bit, we need to: |
| 566 | // 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD operations to |
| 567 | // complete. |
| 568 | // 2. Restore the correct value of vccz by writing the current value |
| 569 | // of vcc back to vcc. |
| 570 | |
| 571 | if (TII->isSMRD(I->getOpcode())) { |
| 572 | VCCZCorrupt = true; |
| 573 | } else if (!hasOutstandingLGKM() && I->modifiesRegister(AMDGPU::VCC, TRI)) { |
| 574 | // FIXME: We only care about SMRD instructions here, not LDS or GDS. |
| 575 | // Whenever we store a value in vcc, the correct value of vccz is |
| 576 | // restored. |
| 577 | VCCZCorrupt = false; |
| 578 | } |
| 579 | |
| 580 | // Check if we need to apply the bug work-around |
| 581 | if (readsVCCZ(I->getOpcode()) && VCCZCorrupt) { |
| 582 | DEBUG(dbgs() << "Inserting vccz bug work-around before: " << *I << '\n'); |
| 583 | |
| 584 | // Wait on everything, not just LGKM. vccz reads usually come from |
| 585 | // terminators, and we always wait on everything at the end of the |
| 586 | // block, so if we only wait on LGKM here, we might end up with |
| 587 | // another s_waitcnt inserted right after this if there are non-LGKM |
| 588 | // instructions still outstanding. |
| 589 | insertWait(MBB, I, LastIssued); |
| 590 | |
| 591 | // Restore the vccz bit. Any time a value is written to vcc, the vcc |
| 592 | // bit is updated, so we can restore the bit by reading the value of |
| 593 | // vcc and then writing it back to the register. |
| 594 | BuildMI(MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64), |
| 595 | AMDGPU::VCC) |
| 596 | .addReg(AMDGPU::VCC); |
| 597 | } |
| 598 | } |
| 599 | |
Tom Stellard | 331f981 | 2016-03-14 17:05:56 +0000 | [diff] [blame] | 600 | if (TII->isDPP(*I)) { |
| 601 | insertDPPWaitStates(I); |
| 602 | } |
| 603 | |
Tom Stellard | 9d6797a | 2015-01-06 19:52:07 +0000 | [diff] [blame] | 604 | // Wait for everything before a barrier. |
| 605 | if (I->getOpcode() == AMDGPU::S_BARRIER) |
| 606 | Changes |= insertWait(MBB, I, LastIssued); |
| 607 | else |
| 608 | Changes |= insertWait(MBB, I, handleOperands(*I)); |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 609 | |
Marek Olsak | fa58e5e | 2014-12-07 17:17:43 +0000 | [diff] [blame] | 610 | pushInstruction(MBB, I); |
Marek Olsak | 1bd2463 | 2015-02-03 17:37:52 +0000 | [diff] [blame] | 611 | handleSendMsg(MBB, I); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 612 | } |
| 613 | |
| 614 | // Wait for everything at the end of the MBB |
| 615 | Changes |= insertWait(MBB, MBB.getFirstTerminator(), LastIssued); |
Tom Stellard | c4cabef | 2013-01-18 21:15:53 +0000 | [diff] [blame] | 616 | } |
| 617 | |
| 618 | return Changes; |
| 619 | } |