blob: 7df43eeb17e8133c193bef0e16ddd5063559f94e [file] [log] [blame]
Tom Stellardc4cabef2013-01-18 21:15:53 +00001//===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// \brief Insert wait instructions for memory reads and writes.
12///
13/// Memory reads and writes are issued asynchronously, so we need to insert
14/// S_WAITCNT instructions when we want to access any of their results or
15/// overwrite any register that's used asynchronously.
16//
17//===----------------------------------------------------------------------===//
18
19#include "AMDGPU.h"
Eric Christopherd9134482014-08-04 21:25:23 +000020#include "AMDGPUSubtarget.h"
Matt Arsenault9783e002014-09-29 15:50:26 +000021#include "SIDefines.h"
Matt Arsenault1fd0c622014-09-29 15:53:15 +000022#include "SIInstrInfo.h"
Tom Stellardc4cabef2013-01-18 21:15:53 +000023#include "SIMachineFunctionInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineFunctionPass.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28
Tom Stellard6e1967e2016-02-05 17:42:38 +000029#define DEBUG_TYPE "si-insert-waits"
30
Tom Stellardc4cabef2013-01-18 21:15:53 +000031using namespace llvm;
32
33namespace {
34
35/// \brief One variable for each of the hardware counters
36typedef union {
37 struct {
38 unsigned VM;
39 unsigned EXP;
40 unsigned LGKM;
41 } Named;
42 unsigned Array[3];
43
44} Counters;
45
Marek Olsakfa58e5e2014-12-07 17:17:43 +000046typedef enum {
47 OTHER,
48 SMEM,
49 VMEM
50} InstType;
51
Tom Stellardc4cabef2013-01-18 21:15:53 +000052typedef Counters RegCounters[512];
53typedef std::pair<unsigned, unsigned> RegInterval;
54
55class SIInsertWaits : public MachineFunctionPass {
56
57private:
Tom Stellardc4cabef2013-01-18 21:15:53 +000058 const SIInstrInfo *TII;
Bill Wendling37e9adb2013-06-07 20:28:55 +000059 const SIRegisterInfo *TRI;
Tom Stellardc4cabef2013-01-18 21:15:53 +000060 const MachineRegisterInfo *MRI;
61
62 /// \brief Constant hardware limits
63 static const Counters WaitCounts;
64
65 /// \brief Constant zero value
66 static const Counters ZeroCounts;
67
68 /// \brief Counter values we have already waited on.
69 Counters WaitedOn;
70
71 /// \brief Counter values for last instruction issued.
72 Counters LastIssued;
73
74 /// \brief Registers used by async instructions.
75 RegCounters UsedRegs;
76
77 /// \brief Registers defined by async instructions.
78 RegCounters DefinedRegs;
79
80 /// \brief Different export instruction types seen since last wait.
81 unsigned ExpInstrTypesSeen;
82
Marek Olsakfa58e5e2014-12-07 17:17:43 +000083 /// \brief Type of the last opcode.
84 InstType LastOpcodeType;
85
Marek Olsak1bd24632015-02-03 17:37:52 +000086 bool LastInstWritesM0;
87
Marek Olsak8e9cc632016-01-13 17:23:09 +000088 /// \brief Whether the machine function returns void
89 bool ReturnsVoid;
90
Tom Stellard30961762016-02-08 19:49:20 +000091 /// Whether the VCCZ bit is possibly corrupt
92 bool VCCZCorrupt;
93
Tom Stellardc4cabef2013-01-18 21:15:53 +000094 /// \brief Get increment/decrement amount for this instruction.
95 Counters getHwCounts(MachineInstr &MI);
96
97 /// \brief Is operand relevant for async execution?
98 bool isOpRelevant(MachineOperand &Op);
99
100 /// \brief Get register interval an operand affects.
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000101 RegInterval getRegInterval(const TargetRegisterClass *RC,
102 const MachineOperand &Reg) const;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000103
104 /// \brief Handle instructions async components
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000105 void pushInstruction(MachineBasicBlock &MBB,
106 MachineBasicBlock::iterator I);
Tom Stellardc4cabef2013-01-18 21:15:53 +0000107
108 /// \brief Insert the actual wait instruction
109 bool insertWait(MachineBasicBlock &MBB,
110 MachineBasicBlock::iterator I,
111 const Counters &Counts);
112
Christian Konig862fd9f2013-03-01 09:46:04 +0000113 /// \brief Do we need def2def checks?
114 bool unorderedDefines(MachineInstr &MI);
115
Tom Stellardc4cabef2013-01-18 21:15:53 +0000116 /// \brief Resolve all operand dependencies to counter requirements
117 Counters handleOperands(MachineInstr &MI);
118
Marek Olsak1bd24632015-02-03 17:37:52 +0000119 /// \brief Insert S_NOP between an instruction writing M0 and S_SENDMSG.
120 void handleSendMsg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I);
121
Tom Stellard331f9812016-03-14 17:05:56 +0000122 /// \param DPP The DPP instruction
123 /// \param SearchI The iterator to start look for hazards.
124 /// \param SearchMBB The basic block we are operating on.
125 /// \param WaitStates Then number of wait states that need to be inserted
126 /// When a hazard is detected.
127 void insertDPPWaitStates(MachineBasicBlock::iterator DPP,
128 MachineBasicBlock::reverse_iterator SearchI,
129 MachineBasicBlock *SearchMBB,
130 unsigned WaitStates);
131
132 void insertDPPWaitStates(MachineBasicBlock::iterator DPP);
133
Tom Stellard30961762016-02-08 19:49:20 +0000134 /// Return true if there are LGKM instrucitons that haven't been waited on
135 /// yet.
136 bool hasOutstandingLGKM() const;
137
Tom Stellardc4cabef2013-01-18 21:15:53 +0000138public:
Tom Stellard6e1967e2016-02-05 17:42:38 +0000139 static char ID;
140
141 SIInsertWaits() :
Tom Stellardc4cabef2013-01-18 21:15:53 +0000142 MachineFunctionPass(ID),
Craig Topper062a2ba2014-04-25 05:30:21 +0000143 TII(nullptr),
144 TRI(nullptr),
Tom Stellard30961762016-02-08 19:49:20 +0000145 ExpInstrTypesSeen(0),
146 VCCZCorrupt(false) { }
Tom Stellardc4cabef2013-01-18 21:15:53 +0000147
Craig Topper5656db42014-04-29 07:57:24 +0000148 bool runOnMachineFunction(MachineFunction &MF) override;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000149
Craig Topper5656db42014-04-29 07:57:24 +0000150 const char *getPassName() const override {
Matt Arsenault0cb85172015-09-25 17:21:28 +0000151 return "SI insert wait instructions";
Tom Stellardc4cabef2013-01-18 21:15:53 +0000152 }
153
Matt Arsenault0cb85172015-09-25 17:21:28 +0000154 void getAnalysisUsage(AnalysisUsage &AU) const override {
155 AU.setPreservesCFG();
156 MachineFunctionPass::getAnalysisUsage(AU);
157 }
Tom Stellardc4cabef2013-01-18 21:15:53 +0000158};
159
160} // End anonymous namespace
161
Tom Stellard6e1967e2016-02-05 17:42:38 +0000162INITIALIZE_PASS_BEGIN(SIInsertWaits, DEBUG_TYPE,
163 "SI Insert Waits", false, false)
164INITIALIZE_PASS_END(SIInsertWaits, DEBUG_TYPE,
165 "SI Insert Waits", false, false)
166
Tom Stellardc4cabef2013-01-18 21:15:53 +0000167char SIInsertWaits::ID = 0;
168
Tom Stellard6e1967e2016-02-05 17:42:38 +0000169char &llvm::SIInsertWaitsID = SIInsertWaits::ID;
170
171FunctionPass *llvm::createSIInsertWaitsPass() {
172 return new SIInsertWaits();
173}
174
Tom Stellard3d2c8522016-01-28 17:13:44 +0000175const Counters SIInsertWaits::WaitCounts = { { 15, 7, 15 } };
Tom Stellardc4cabef2013-01-18 21:15:53 +0000176const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } };
177
Tom Stellard30961762016-02-08 19:49:20 +0000178static bool readsVCCZ(unsigned Opcode) {
Matt Arsenaultf2dcb472016-03-02 04:12:39 +0000179 return Opcode == AMDGPU::S_CBRANCH_VCCNZ || Opcode == AMDGPU::S_CBRANCH_VCCZ;
Tom Stellard30961762016-02-08 19:49:20 +0000180}
181
182bool SIInsertWaits::hasOutstandingLGKM() const {
183 return WaitedOn.Named.LGKM != LastIssued.Named.LGKM;
184}
Tom Stellardc4cabef2013-01-18 21:15:53 +0000185
186Counters SIInsertWaits::getHwCounts(MachineInstr &MI) {
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000187 uint64_t TSFlags = MI.getDesc().TSFlags;
Matt Arsenaulte66621b2015-09-24 19:52:27 +0000188 Counters Result = { { 0, 0, 0 } };
Tom Stellardc4cabef2013-01-18 21:15:53 +0000189
190 Result.Named.VM = !!(TSFlags & SIInstrFlags::VM_CNT);
191
192 // Only consider stores or EXP for EXP_CNT
193 Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT &&
Christian Konig862fd9f2013-03-01 09:46:04 +0000194 (MI.getOpcode() == AMDGPU::EXP || MI.getDesc().mayStore()));
Tom Stellardc4cabef2013-01-18 21:15:53 +0000195
196 // LGKM may uses larger values
197 if (TSFlags & SIInstrFlags::LGKM_CNT) {
198
Matt Arsenault3add6432015-10-20 04:35:43 +0000199 if (TII->isSMRD(MI)) {
Tom Stellardc4cabef2013-01-18 21:15:53 +0000200
Matt Arsenaulte66621b2015-09-24 19:52:27 +0000201 if (MI.getNumOperands() != 0) {
Matt Arsenaultb733f002015-10-01 22:40:35 +0000202 assert(MI.getOperand(0).isReg() &&
203 "First LGKM operand must be a register!");
Michel Danzer20680b12013-08-16 16:19:24 +0000204
Matt Arsenaulte66621b2015-09-24 19:52:27 +0000205 // XXX - What if this is a write into a super register?
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000206 const TargetRegisterClass *RC = TII->getOpRegClass(MI, 0);
207 unsigned Size = RC->getSize();
Matt Arsenaulte66621b2015-09-24 19:52:27 +0000208 Result.Named.LGKM = Size > 4 ? 2 : 1;
209 } else {
210 // s_dcache_inv etc. do not have a a destination register. Assume we
211 // want a wait on these.
212 // XXX - What is the right value?
213 Result.Named.LGKM = 1;
214 }
Michel Danzer20680b12013-08-16 16:19:24 +0000215 } else {
216 // DS
217 Result.Named.LGKM = 1;
218 }
Tom Stellardc4cabef2013-01-18 21:15:53 +0000219
220 } else {
221 Result.Named.LGKM = 0;
222 }
223
224 return Result;
225}
226
227bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
Tom Stellardc4cabef2013-01-18 21:15:53 +0000228 // Constants are always irrelevant
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000229 if (!Op.isReg() || !TRI->isInAllocatableClass(Op.getReg()))
Tom Stellardc4cabef2013-01-18 21:15:53 +0000230 return false;
231
232 // Defines are always relevant
233 if (Op.isDef())
234 return true;
235
236 // For exports all registers are relevant
237 MachineInstr &MI = *Op.getParent();
238 if (MI.getOpcode() == AMDGPU::EXP)
239 return true;
240
241 // For stores the stored value is also relevant
242 if (!MI.getDesc().mayStore())
243 return false;
244
Tom Stellardb3931b82015-01-06 19:52:04 +0000245 // Check if this operand is the value being stored.
Tom Stellard2d26fe72016-02-19 15:33:13 +0000246 // Special case for DS/FLAT instructions, since the address
Tom Stellardb3931b82015-01-06 19:52:04 +0000247 // operand comes before the value operand and it may have
248 // multiple data operands.
249
Tom Stellard2d26fe72016-02-19 15:33:13 +0000250 if (TII->isDS(MI) || TII->isFLAT(MI)) {
Tom Stellardb3931b82015-01-06 19:52:04 +0000251 MachineOperand *Data = TII->getNamedOperand(MI, AMDGPU::OpName::data);
252 if (Data && Op.isIdenticalTo(*Data))
253 return true;
Tom Stellard2d26fe72016-02-19 15:33:13 +0000254 }
Tom Stellardb3931b82015-01-06 19:52:04 +0000255
Tom Stellard2d26fe72016-02-19 15:33:13 +0000256 if (TII->isDS(MI)) {
Tom Stellardb3931b82015-01-06 19:52:04 +0000257 MachineOperand *Data0 = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
258 if (Data0 && Op.isIdenticalTo(*Data0))
259 return true;
260
261 MachineOperand *Data1 = TII->getNamedOperand(MI, AMDGPU::OpName::data1);
Matt Arsenault8226fc42016-03-02 23:00:21 +0000262 return Data1 && Op.isIdenticalTo(*Data1);
Tom Stellardb3931b82015-01-06 19:52:04 +0000263 }
264
265 // NOTE: This assumes that the value operand is before the
266 // address operand, and that there is only one value operand.
Tom Stellardc4cabef2013-01-18 21:15:53 +0000267 for (MachineInstr::mop_iterator I = MI.operands_begin(),
268 E = MI.operands_end(); I != E; ++I) {
269
270 if (I->isReg() && I->isUse())
271 return Op.isIdenticalTo(*I);
272 }
273
274 return false;
275}
276
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000277RegInterval SIInsertWaits::getRegInterval(const TargetRegisterClass *RC,
278 const MachineOperand &Reg) const {
279 unsigned Size = RC->getSize();
Tom Stellardc4cabef2013-01-18 21:15:53 +0000280 assert(Size >= 4);
281
282 RegInterval Result;
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000283 Result.first = TRI->getEncodingValue(Reg.getReg());
Tom Stellardc4cabef2013-01-18 21:15:53 +0000284 Result.second = Result.first + Size / 4;
285
286 return Result;
287}
288
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000289void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB,
290 MachineBasicBlock::iterator I) {
Tom Stellardc4cabef2013-01-18 21:15:53 +0000291
292 // Get the hardware counter increments and sum them up
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000293 Counters Increment = getHwCounts(*I);
Tom Stellardbd8a0852015-08-21 22:47:27 +0000294 Counters Limit = ZeroCounts;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000295 unsigned Sum = 0;
296
297 for (unsigned i = 0; i < 3; ++i) {
298 LastIssued.Array[i] += Increment.Array[i];
Tom Stellardbd8a0852015-08-21 22:47:27 +0000299 if (Increment.Array[i])
300 Limit.Array[i] = LastIssued.Array[i];
Tom Stellardc4cabef2013-01-18 21:15:53 +0000301 Sum += Increment.Array[i];
302 }
303
304 // If we don't increase anything then that's it
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000305 if (Sum == 0) {
306 LastOpcodeType = OTHER;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000307 return;
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000308 }
309
Eric Christopher6c5b5112015-03-11 18:43:21 +0000310 if (MBB.getParent()->getSubtarget<AMDGPUSubtarget>().getGeneration() >=
311 AMDGPUSubtarget::VOLCANIC_ISLANDS) {
Benjamin Kramerdf005cb2015-08-08 18:27:36 +0000312 // Any occurrence of consecutive VMEM or SMEM instructions forms a VMEM
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000313 // or SMEM clause, respectively.
314 //
315 // The temporary workaround is to break the clauses with S_NOP.
316 //
317 // The proper solution would be to allocate registers such that all source
318 // and destination registers don't overlap, e.g. this is illegal:
319 // r0 = load r2
320 // r2 = load r0
Matt Arsenault3add6432015-10-20 04:35:43 +0000321 if ((LastOpcodeType == SMEM && TII->isSMRD(*I)) ||
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000322 (LastOpcodeType == VMEM && Increment.Named.VM)) {
323 // Insert a NOP to break the clause.
324 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP))
325 .addImm(0);
Marek Olsak1bd24632015-02-03 17:37:52 +0000326 LastInstWritesM0 = false;
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000327 }
328
Matt Arsenault3add6432015-10-20 04:35:43 +0000329 if (TII->isSMRD(*I))
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000330 LastOpcodeType = SMEM;
331 else if (Increment.Named.VM)
332 LastOpcodeType = VMEM;
333 }
Tom Stellardc4cabef2013-01-18 21:15:53 +0000334
335 // Remember which export instructions we have seen
336 if (Increment.Named.EXP) {
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000337 ExpInstrTypesSeen |= I->getOpcode() == AMDGPU::EXP ? 1 : 2;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000338 }
339
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000340 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000341 MachineOperand &Op = I->getOperand(i);
Tom Stellardc4cabef2013-01-18 21:15:53 +0000342 if (!isOpRelevant(Op))
343 continue;
344
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000345 const TargetRegisterClass *RC = TII->getOpRegClass(*I, i);
346 RegInterval Interval = getRegInterval(RC, Op);
Tom Stellardc4cabef2013-01-18 21:15:53 +0000347 for (unsigned j = Interval.first; j < Interval.second; ++j) {
348
349 // Remember which registers we define
350 if (Op.isDef())
Tom Stellardbd8a0852015-08-21 22:47:27 +0000351 DefinedRegs[j] = Limit;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000352
353 // and which one we are using
354 if (Op.isUse())
Tom Stellardbd8a0852015-08-21 22:47:27 +0000355 UsedRegs[j] = Limit;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000356 }
357 }
358}
359
360bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
361 MachineBasicBlock::iterator I,
362 const Counters &Required) {
363
364 // End of program? No need to wait on anything
Marek Olsak8e9cc632016-01-13 17:23:09 +0000365 // A function not returning void needs to wait, because other bytecode will
366 // be appended after it and we don't know what it will be.
367 if (I != MBB.end() && I->getOpcode() == AMDGPU::S_ENDPGM && ReturnsVoid)
Tom Stellardc4cabef2013-01-18 21:15:53 +0000368 return false;
369
370 // Figure out if the async instructions execute in order
371 bool Ordered[3];
372
373 // VM_CNT is always ordered
374 Ordered[0] = true;
375
376 // EXP_CNT is unordered if we have both EXP & VM-writes
377 Ordered[1] = ExpInstrTypesSeen == 3;
378
379 // LGKM_CNT is handled as always unordered. TODO: Handle LDS and GDS
380 Ordered[2] = false;
381
382 // The values we are going to put into the S_WAITCNT instruction
383 Counters Counts = WaitCounts;
384
385 // Do we really need to wait?
386 bool NeedWait = false;
387
388 for (unsigned i = 0; i < 3; ++i) {
389
390 if (Required.Array[i] <= WaitedOn.Array[i])
391 continue;
392
393 NeedWait = true;
Matt Arsenault97483692014-07-17 17:50:22 +0000394
Tom Stellardc4cabef2013-01-18 21:15:53 +0000395 if (Ordered[i]) {
396 unsigned Value = LastIssued.Array[i] - Required.Array[i];
397
Matt Arsenault97483692014-07-17 17:50:22 +0000398 // Adjust the value to the real hardware possibilities.
Tom Stellardc4cabef2013-01-18 21:15:53 +0000399 Counts.Array[i] = std::min(Value, WaitCounts.Array[i]);
400
401 } else
402 Counts.Array[i] = 0;
403
Matt Arsenault97483692014-07-17 17:50:22 +0000404 // Remember on what we have waited on.
Tom Stellardc4cabef2013-01-18 21:15:53 +0000405 WaitedOn.Array[i] = LastIssued.Array[i] - Counts.Array[i];
406 }
407
408 if (!NeedWait)
409 return false;
410
411 // Reset EXP_CNT instruction types
412 if (Counts.Named.EXP == 0)
413 ExpInstrTypesSeen = 0;
414
415 // Build the wait instruction
416 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
417 .addImm((Counts.Named.VM & 0xF) |
418 ((Counts.Named.EXP & 0x7) << 4) |
Tom Stellard3d2c8522016-01-28 17:13:44 +0000419 ((Counts.Named.LGKM & 0xF) << 8));
Tom Stellardc4cabef2013-01-18 21:15:53 +0000420
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000421 LastOpcodeType = OTHER;
Marek Olsak1bd24632015-02-03 17:37:52 +0000422 LastInstWritesM0 = false;
Tom Stellardc4cabef2013-01-18 21:15:53 +0000423 return true;
424}
425
426/// \brief helper function for handleOperands
427static void increaseCounters(Counters &Dst, const Counters &Src) {
428
429 for (unsigned i = 0; i < 3; ++i)
430 Dst.Array[i] = std::max(Dst.Array[i], Src.Array[i]);
431}
432
433Counters SIInsertWaits::handleOperands(MachineInstr &MI) {
434
435 Counters Result = ZeroCounts;
436
Michel Danzer6064f572014-01-27 07:20:44 +0000437 // S_SENDMSG implicitly waits for all outstanding LGKM transfers to finish,
438 // but we also want to wait for any other outstanding transfers before
439 // signalling other hardware blocks
440 if (MI.getOpcode() == AMDGPU::S_SENDMSG)
441 return LastIssued;
442
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000443 // For each register affected by this instruction increase the result
444 // sequence.
445 //
446 // TODO: We could probably just look at explicit operands if we removed VCC /
447 // EXEC from SMRD dest reg classes.
Tom Stellardc4cabef2013-01-18 21:15:53 +0000448 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
Tom Stellardc4cabef2013-01-18 21:15:53 +0000449 MachineOperand &Op = MI.getOperand(i);
Matt Arsenaultd1d499a2015-10-01 21:43:15 +0000450 if (!Op.isReg() || !TRI->isInAllocatableClass(Op.getReg()))
451 continue;
452
453 const TargetRegisterClass *RC = TII->getOpRegClass(MI, i);
454 RegInterval Interval = getRegInterval(RC, Op);
Tom Stellardc4cabef2013-01-18 21:15:53 +0000455 for (unsigned j = Interval.first; j < Interval.second; ++j) {
456
Christian Konig862fd9f2013-03-01 09:46:04 +0000457 if (Op.isDef()) {
Tom Stellardc4cabef2013-01-18 21:15:53 +0000458 increaseCounters(Result, UsedRegs[j]);
Christian Konigf1fd5fa2013-03-18 11:33:45 +0000459 increaseCounters(Result, DefinedRegs[j]);
Christian Konig862fd9f2013-03-01 09:46:04 +0000460 }
Tom Stellardc4cabef2013-01-18 21:15:53 +0000461
462 if (Op.isUse())
463 increaseCounters(Result, DefinedRegs[j]);
464 }
465 }
466
467 return Result;
468}
469
Marek Olsak1bd24632015-02-03 17:37:52 +0000470void SIInsertWaits::handleSendMsg(MachineBasicBlock &MBB,
471 MachineBasicBlock::iterator I) {
Eric Christopher6c5b5112015-03-11 18:43:21 +0000472 if (MBB.getParent()->getSubtarget<AMDGPUSubtarget>().getGeneration() <
473 AMDGPUSubtarget::VOLCANIC_ISLANDS)
Marek Olsak1bd24632015-02-03 17:37:52 +0000474 return;
475
476 // There must be "S_NOP 0" between an instruction writing M0 and S_SENDMSG.
477 if (LastInstWritesM0 && I->getOpcode() == AMDGPU::S_SENDMSG) {
478 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP)).addImm(0);
479 LastInstWritesM0 = false;
480 return;
481 }
482
483 // Set whether this instruction sets M0
484 LastInstWritesM0 = false;
485
486 unsigned NumOperands = I->getNumOperands();
487 for (unsigned i = 0; i < NumOperands; i++) {
488 const MachineOperand &Op = I->getOperand(i);
489
490 if (Op.isReg() && Op.isDef() && Op.getReg() == AMDGPU::M0)
491 LastInstWritesM0 = true;
492 }
493}
494
Tom Stellard331f9812016-03-14 17:05:56 +0000495void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP,
496 MachineBasicBlock::reverse_iterator SearchI,
497 MachineBasicBlock *SearchMBB,
498 unsigned WaitStates) {
499
500 MachineBasicBlock::reverse_iterator E = SearchMBB->rend();
501
502 for (; WaitStates > 0; --WaitStates, ++SearchI) {
503
504 // If we have reached the start of the block, we need to check predecessors.
505 if (SearchI == E) {
506 for (MachineBasicBlock *Pred : SearchMBB->predecessors()) {
507 // We only need to check fall-through blocks. Branch instructions
508 // give us enough wait states.
509 if (Pred->getFirstTerminator() == Pred->end()) {
510 insertDPPWaitStates(DPP, Pred->rbegin(), Pred, WaitStates);
511 break;
512 }
513 }
514 return;
515 }
516
517 for (MachineOperand &Op : SearchI->operands()) {
518 if (!Op.isReg() || !Op.isDef())
519 continue;
520
521 if (DPP->readsRegister(Op.getReg(), TRI)) {
522 TII->insertWaitStates(DPP, WaitStates);
523 return;
524 }
525 }
526 }
527}
528
529void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP) {
530 MachineBasicBlock::reverse_iterator I(DPP);
531 insertDPPWaitStates(DPP, I, DPP->getParent(), 2);
532}
533
Matt Arsenaulta0050b02014-06-19 01:19:19 +0000534// FIXME: Insert waits listed in Table 4.2 "Required User-Inserted Wait States"
535// around other non-memory instructions.
Tom Stellardc4cabef2013-01-18 21:15:53 +0000536bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
Tom Stellardc4cabef2013-01-18 21:15:53 +0000537 bool Changes = false;
538
Eric Christopherfc6de422014-08-05 02:39:49 +0000539 TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
540 TRI =
541 static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo());
Bill Wendling37e9adb2013-06-07 20:28:55 +0000542
Tom Stellard30961762016-02-08 19:49:20 +0000543 const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
Tom Stellardc4cabef2013-01-18 21:15:53 +0000544 MRI = &MF.getRegInfo();
545
546 WaitedOn = ZeroCounts;
547 LastIssued = ZeroCounts;
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000548 LastOpcodeType = OTHER;
Marek Olsak1bd24632015-02-03 17:37:52 +0000549 LastInstWritesM0 = false;
Marek Olsak8e9cc632016-01-13 17:23:09 +0000550 ReturnsVoid = MF.getInfo<SIMachineFunctionInfo>()->returnsVoid();
Tom Stellardc4cabef2013-01-18 21:15:53 +0000551
552 memset(&UsedRegs, 0, sizeof(UsedRegs));
553 memset(&DefinedRegs, 0, sizeof(DefinedRegs));
554
555 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
556 BI != BE; ++BI) {
557
558 MachineBasicBlock &MBB = *BI;
559 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
560 I != E; ++I) {
561
Tom Stellard30961762016-02-08 19:49:20 +0000562 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) {
563 // There is a hardware bug on CI/SI where SMRD instruction may corrupt
564 // vccz bit, so when we detect that an instruction may read from a
565 // corrupt vccz bit, we need to:
566 // 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD operations to
567 // complete.
568 // 2. Restore the correct value of vccz by writing the current value
569 // of vcc back to vcc.
570
571 if (TII->isSMRD(I->getOpcode())) {
572 VCCZCorrupt = true;
573 } else if (!hasOutstandingLGKM() && I->modifiesRegister(AMDGPU::VCC, TRI)) {
574 // FIXME: We only care about SMRD instructions here, not LDS or GDS.
575 // Whenever we store a value in vcc, the correct value of vccz is
576 // restored.
577 VCCZCorrupt = false;
578 }
579
580 // Check if we need to apply the bug work-around
581 if (readsVCCZ(I->getOpcode()) && VCCZCorrupt) {
582 DEBUG(dbgs() << "Inserting vccz bug work-around before: " << *I << '\n');
583
584 // Wait on everything, not just LGKM. vccz reads usually come from
585 // terminators, and we always wait on everything at the end of the
586 // block, so if we only wait on LGKM here, we might end up with
587 // another s_waitcnt inserted right after this if there are non-LGKM
588 // instructions still outstanding.
589 insertWait(MBB, I, LastIssued);
590
591 // Restore the vccz bit. Any time a value is written to vcc, the vcc
592 // bit is updated, so we can restore the bit by reading the value of
593 // vcc and then writing it back to the register.
594 BuildMI(MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
595 AMDGPU::VCC)
596 .addReg(AMDGPU::VCC);
597 }
598 }
599
Tom Stellard331f9812016-03-14 17:05:56 +0000600 if (TII->isDPP(*I)) {
601 insertDPPWaitStates(I);
602 }
603
Tom Stellard9d6797a2015-01-06 19:52:07 +0000604 // Wait for everything before a barrier.
605 if (I->getOpcode() == AMDGPU::S_BARRIER)
606 Changes |= insertWait(MBB, I, LastIssued);
607 else
608 Changes |= insertWait(MBB, I, handleOperands(*I));
Marek Olsak1bd24632015-02-03 17:37:52 +0000609
Marek Olsakfa58e5e2014-12-07 17:17:43 +0000610 pushInstruction(MBB, I);
Marek Olsak1bd24632015-02-03 17:37:52 +0000611 handleSendMsg(MBB, I);
Tom Stellardc4cabef2013-01-18 21:15:53 +0000612 }
613
614 // Wait for everything at the end of the MBB
615 Changes |= insertWait(MBB, MBB.getFirstTerminator(), LastIssued);
Tom Stellardc4cabef2013-01-18 21:15:53 +0000616 }
617
618 return Changes;
619}