blob: 76b0b4573fc5809864eff5bfe52caf7ed099c8c2 [file] [log] [blame]
Nicolai Haehnle213e87f2016-03-21 20:28:33 +00001//===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// \brief This pass adds instructions to enable whole quad mode for pixel
12/// shaders.
13///
14/// Whole quad mode is required for derivative computations, but it interferes
15/// with shader side effects (stores and atomics). This pass is run on the
16/// scheduled machine IR but before register coalescing, so that machine SSA is
17/// available for analysis. It ensures that WQM is enabled when necessary, but
18/// disabled around stores and atomics.
19///
20/// When necessary, this pass creates a function prolog
21///
22/// S_MOV_B64 LiveMask, EXEC
23/// S_WQM_B64 EXEC, EXEC
24///
25/// to enter WQM at the top of the function and surrounds blocks of Exact
26/// instructions by
27///
28/// S_AND_SAVEEXEC_B64 Tmp, LiveMask
29/// ...
30/// S_MOV_B64 EXEC, Tmp
31///
32/// In order to avoid excessive switching during sequences of Exact
33/// instructions, the pass first analyzes which instructions must be run in WQM
34/// (aka which instructions produce values that lead to derivative
35/// computations).
36///
37/// Basic blocks are always exited in WQM as long as some successor needs WQM.
38///
39/// There is room for improvement given better control flow analysis:
40///
41/// (1) at the top level (outside of control flow statements, and as long as
42/// kill hasn't been used), one SGPR can be saved by recovering WQM from
43/// the LiveMask (this is implemented for the entry block).
44///
45/// (2) when entire regions (e.g. if-else blocks or entire loops) only
46/// consist of exact and don't-care instructions, the switch only has to
47/// be done at the entry and exit points rather than potentially in each
48/// block of the region.
49///
50//===----------------------------------------------------------------------===//
51
52#include "AMDGPU.h"
53#include "AMDGPUSubtarget.h"
54#include "SIInstrInfo.h"
55#include "SIMachineFunctionInfo.h"
Nicolai Haehnle213e87f2016-03-21 20:28:33 +000056#include "llvm/CodeGen/MachineFunction.h"
57#include "llvm/CodeGen/MachineFunctionPass.h"
58#include "llvm/CodeGen/MachineInstrBuilder.h"
59#include "llvm/CodeGen/MachineRegisterInfo.h"
Nicolai Haehnle213e87f2016-03-21 20:28:33 +000060
61using namespace llvm;
62
63#define DEBUG_TYPE "si-wqm"
64
65namespace {
66
67enum {
68 StateWQM = 0x1,
69 StateExact = 0x2,
70};
71
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +000072struct PrintState {
73public:
74 explicit PrintState(int State) : State(State) {}
75
76 int State;
77};
78
79static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
80 if (PS.State & StateWQM)
81 OS << "WQM";
82 if (PS.State & StateExact) {
83 if (PS.State & StateWQM)
84 OS << '|';
85 OS << "Exact";
86 }
87
88 return OS;
89}
90
Nicolai Haehnle213e87f2016-03-21 20:28:33 +000091struct InstrInfo {
92 char Needs = 0;
93 char OutNeeds = 0;
94};
95
96struct BlockInfo {
97 char Needs = 0;
98 char InNeeds = 0;
99 char OutNeeds = 0;
100};
101
102struct WorkItem {
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000103 MachineBasicBlock *MBB = nullptr;
104 MachineInstr *MI = nullptr;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000105
106 WorkItem() {}
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000107 WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}
108 WorkItem(MachineInstr *MI) : MI(MI) {}
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000109};
110
111class SIWholeQuadMode : public MachineFunctionPass {
112private:
113 const SIInstrInfo *TII;
114 const SIRegisterInfo *TRI;
115 MachineRegisterInfo *MRI;
Nicolai Haehnlebef0e902016-08-02 19:17:37 +0000116 LiveIntervals *LIS;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000117
118 DenseMap<const MachineInstr *, InstrInfo> Instructions;
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000119 DenseMap<MachineBasicBlock *, BlockInfo> Blocks;
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000120 SmallVector<MachineInstr *, 1> LiveMaskQueries;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000121
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000122 void printInfo();
123
Nicolai Haehnlebef0e902016-08-02 19:17:37 +0000124 void markInstruction(MachineInstr &MI, char Flag,
125 std::vector<WorkItem> &Worklist);
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000126 void markUsesWQM(const MachineInstr &MI, std::vector<WorkItem> &Worklist);
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000127 char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist);
128 void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);
129 void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000130 char analyzeFunction(MachineFunction &MF);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000131
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000132 bool requiresCorrectState(const MachineInstr &MI) const;
133
134 MachineBasicBlock::iterator saveSCC(MachineBasicBlock &MBB,
135 MachineBasicBlock::iterator Before);
136 MachineBasicBlock::iterator
137 prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
138 MachineBasicBlock::iterator Last, bool PreferLast,
139 bool SaveSCC);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000140 void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
141 unsigned SaveWQM, unsigned LiveMaskReg);
142 void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
143 unsigned SavedWQM);
144 void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry);
145
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000146 void lowerLiveMaskQueries(unsigned LiveMaskReg);
147
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000148public:
149 static char ID;
150
151 SIWholeQuadMode() :
152 MachineFunctionPass(ID) { }
153
154 bool runOnMachineFunction(MachineFunction &MF) override;
155
Mehdi Amini117296c2016-10-01 02:56:57 +0000156 StringRef getPassName() const override { return "SI Whole Quad Mode"; }
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000157
158 void getAnalysisUsage(AnalysisUsage &AU) const override {
Nicolai Haehnlebef0e902016-08-02 19:17:37 +0000159 AU.addRequired<LiveIntervals>();
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000160 AU.setPreservesCFG();
161 MachineFunctionPass::getAnalysisUsage(AU);
162 }
163};
164
165} // End anonymous namespace
166
167char SIWholeQuadMode::ID = 0;
168
Nicolai Haehnlebef0e902016-08-02 19:17:37 +0000169INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
170 false)
171INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
172INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
173 false)
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000174
175char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID;
176
177FunctionPass *llvm::createSIWholeQuadModePass() {
178 return new SIWholeQuadMode;
179}
180
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000181void SIWholeQuadMode::printInfo() {
182 for (const auto &BII : Blocks) {
183 dbgs() << "\nBB#" << BII.first->getNumber() << ":\n"
184 << " InNeeds = " << PrintState(BII.second.InNeeds)
185 << ", Needs = " << PrintState(BII.second.Needs)
186 << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n";
187
188 for (const MachineInstr &MI : *BII.first) {
189 auto III = Instructions.find(&MI);
190 if (III == Instructions.end())
191 continue;
192
193 dbgs() << " " << MI << " Needs = " << PrintState(III->second.Needs)
194 << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n';
195 }
196 }
197}
198
Nicolai Haehnlebef0e902016-08-02 19:17:37 +0000199void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag,
200 std::vector<WorkItem> &Worklist) {
201 InstrInfo &II = Instructions[&MI];
202
203 assert(Flag == StateWQM || Flag == StateExact);
204
205 // Ignore if the instruction is already marked. The typical case is that we
206 // mark an instruction WQM multiple times, but for atomics it can happen that
207 // Flag is StateWQM, but Needs is already set to StateExact. In this case,
208 // letting the atomic run in StateExact is correct as per the relevant specs.
209 if (II.Needs)
210 return;
211
212 II.Needs = Flag;
213 Worklist.push_back(&MI);
214}
215
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000216/// Mark all instructions defining the uses in \p MI as WQM.
217void SIWholeQuadMode::markUsesWQM(const MachineInstr &MI,
218 std::vector<WorkItem> &Worklist) {
219 for (const MachineOperand &Use : MI.uses()) {
220 if (!Use.isReg() || !Use.isUse())
221 continue;
222
223 unsigned Reg = Use.getReg();
224
225 // Handle physical registers that we need to track; this is mostly relevant
226 // for VCC, which can appear as the (implicit) input of a uniform branch,
227 // e.g. when a loop counter is stored in a VGPR.
228 if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
229 if (Reg == AMDGPU::EXEC)
230 continue;
231
232 for (MCRegUnitIterator RegUnit(Reg, TRI); RegUnit.isValid(); ++RegUnit) {
233 LiveRange &LR = LIS->getRegUnit(*RegUnit);
234 const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn();
235 if (!Value)
236 continue;
237
238 // Since we're in machine SSA, we do not need to track physical
239 // registers across basic blocks.
240 if (Value->isPHIDef())
241 continue;
242
243 markInstruction(*LIS->getInstructionFromIndex(Value->def), StateWQM,
244 Worklist);
245 }
246
247 continue;
248 }
249
250 for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg()))
251 markInstruction(DefMI, StateWQM, Worklist);
252 }
253}
254
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000255// Scan instructions to determine which ones require an Exact execmask and
256// which ones seed WQM requirements.
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000257char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000258 std::vector<WorkItem> &Worklist) {
259 char GlobalFlags = 0;
Nicolai Haehnlec00e03b2016-06-07 21:37:17 +0000260 bool WQMOutputs = MF.getFunction()->hasFnAttribute("amdgpu-ps-wqm-outputs");
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000261
262 for (auto BI = MF.begin(), BE = MF.end(); BI != BE; ++BI) {
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000263 MachineBasicBlock &MBB = *BI;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000264
265 for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) {
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000266 MachineInstr &MI = *II;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000267 unsigned Opcode = MI.getOpcode();
Nicolai Haehnlec00e03b2016-06-07 21:37:17 +0000268 char Flags = 0;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000269
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000270 if (TII->isDS(Opcode)) {
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000271 Flags = StateWQM;
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000272 } else if (TII->isWQM(Opcode)) {
273 // Sampling instructions don't need to produce results for all pixels
274 // in a quad, they just require all inputs of a quad to have been
275 // computed for derivatives.
276 markUsesWQM(MI, Worklist);
277 GlobalFlags |= StateWQM;
278 continue;
Nicolai Haehnle8a482b32016-08-02 19:31:14 +0000279 } else if (TII->isDisableWQM(MI)) {
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000280 Flags = StateExact;
281 } else {
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000282 if (Opcode == AMDGPU::SI_PS_LIVE) {
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000283 LiveMaskQueries.push_back(&MI);
Nicolai Haehnlec00e03b2016-06-07 21:37:17 +0000284 } else if (WQMOutputs) {
285 // The function is in machine SSA form, which means that physical
286 // VGPRs correspond to shader inputs and outputs. Inputs are
287 // only used, outputs are only defined.
288 for (const MachineOperand &MO : MI.defs()) {
289 if (!MO.isReg())
290 continue;
291
292 unsigned Reg = MO.getReg();
293
294 if (!TRI->isVirtualRegister(Reg) &&
295 TRI->hasVGPRs(TRI->getPhysRegClass(Reg))) {
296 Flags = StateWQM;
297 break;
298 }
299 }
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000300 }
301
Nicolai Haehnlec00e03b2016-06-07 21:37:17 +0000302 if (!Flags)
303 continue;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000304 }
305
Nicolai Haehnlebef0e902016-08-02 19:17:37 +0000306 markInstruction(MI, Flags, Worklist);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000307 GlobalFlags |= Flags;
308 }
309 }
310
311 return GlobalFlags;
312}
313
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000314void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000315 std::vector<WorkItem>& Worklist) {
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000316 MachineBasicBlock *MBB = MI.getParent();
Nicolai Haehnle0a33abd2016-03-21 22:54:02 +0000317 InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000318 BlockInfo &BI = Blocks[MBB];
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000319
Nicolai Haehnle8a482b32016-08-02 19:31:14 +0000320 // Control flow-type instructions and stores to temporary memory that are
321 // followed by WQM computations must themselves be in WQM.
322 if ((II.OutNeeds & StateWQM) && !II.Needs &&
323 (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) {
Nicolai Haehnle0a33abd2016-03-21 22:54:02 +0000324 Instructions[&MI].Needs = StateWQM;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000325 II.Needs = StateWQM;
Nicolai Haehnle0a33abd2016-03-21 22:54:02 +0000326 }
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000327
328 // Propagate to block level
329 BI.Needs |= II.Needs;
330 if ((BI.InNeeds | II.Needs) != BI.InNeeds) {
331 BI.InNeeds |= II.Needs;
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000332 Worklist.push_back(MBB);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000333 }
334
335 // Propagate backwards within block
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000336 if (MachineInstr *PrevMI = MI.getPrevNode()) {
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000337 char InNeeds = II.Needs | II.OutNeeds;
338 if (!PrevMI->isPHI()) {
339 InstrInfo &PrevII = Instructions[PrevMI];
340 if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
341 PrevII.OutNeeds |= InNeeds;
342 Worklist.push_back(PrevMI);
343 }
344 }
345 }
346
347 // Propagate WQM flag to instruction inputs
348 assert(II.Needs != (StateWQM | StateExact));
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000349
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000350 if (II.Needs == StateWQM)
351 markUsesWQM(MI, Worklist);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000352}
353
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000354void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000355 std::vector<WorkItem>& Worklist) {
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000356 BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000357
358 // Propagate through instructions
359 if (!MBB.empty()) {
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000360 MachineInstr *LastMI = &*MBB.rbegin();
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000361 InstrInfo &LastII = Instructions[LastMI];
362 if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
363 LastII.OutNeeds |= BI.OutNeeds;
364 Worklist.push_back(LastMI);
365 }
366 }
367
368 // Predecessor blocks must provide for our WQM/Exact needs.
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000369 for (MachineBasicBlock *Pred : MBB.predecessors()) {
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000370 BlockInfo &PredBI = Blocks[Pred];
371 if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
372 continue;
373
374 PredBI.OutNeeds |= BI.InNeeds;
375 PredBI.InNeeds |= BI.InNeeds;
376 Worklist.push_back(Pred);
377 }
378
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000379 // All successors must be prepared to accept the same set of WQM/Exact data.
380 for (MachineBasicBlock *Succ : MBB.successors()) {
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000381 BlockInfo &SuccBI = Blocks[Succ];
382 if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
383 continue;
384
385 SuccBI.InNeeds |= BI.OutNeeds;
386 Worklist.push_back(Succ);
387 }
388}
389
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000390char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) {
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000391 std::vector<WorkItem> Worklist;
392 char GlobalFlags = scanInstructions(MF, Worklist);
393
394 while (!Worklist.empty()) {
395 WorkItem WI = Worklist.back();
396 Worklist.pop_back();
397
398 if (WI.MI)
399 propagateInstruction(*WI.MI, Worklist);
400 else
401 propagateBlock(*WI.MBB, Worklist);
402 }
403
404 return GlobalFlags;
405}
406
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000407/// Whether \p MI really requires the exec state computed during analysis.
408///
409/// Scalar instructions must occasionally be marked WQM for correct propagation
410/// (e.g. thread masks leading up to branches), but when it comes to actual
411/// execution, they don't care about EXEC.
412bool SIWholeQuadMode::requiresCorrectState(const MachineInstr &MI) const {
413 if (MI.isTerminator())
414 return true;
415
416 // Skip instructions that are not affected by EXEC
417 if (TII->isScalarUnit(MI))
418 return false;
419
420 // Generic instructions such as COPY will either disappear by register
421 // coalescing or be lowered to SALU or VALU instructions.
422 if (MI.isTransient()) {
423 if (MI.getNumExplicitOperands() >= 1) {
424 const MachineOperand &Op = MI.getOperand(0);
425 if (Op.isReg()) {
426 if (TRI->isSGPRReg(*MRI, Op.getReg())) {
427 // SGPR instructions are not affected by EXEC
428 return false;
429 }
430 }
431 }
432 }
433
434 return true;
435}
436
437MachineBasicBlock::iterator
438SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,
439 MachineBasicBlock::iterator Before) {
Marek Olsak79c05872016-11-25 17:37:09 +0000440 unsigned SaveReg = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000441
442 MachineInstr *Save =
443 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), SaveReg)
444 .addReg(AMDGPU::SCC);
445 MachineInstr *Restore =
446 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::SCC)
447 .addReg(SaveReg);
448
449 LIS->InsertMachineInstrInMaps(*Save);
450 LIS->InsertMachineInstrInMaps(*Restore);
451 LIS->createAndComputeVirtRegInterval(SaveReg);
452
453 return Restore;
454}
455
456// Return an iterator in the (inclusive) range [First, Last] at which
457// instructions can be safely inserted, keeping in mind that some of the
458// instructions we want to add necessarily clobber SCC.
459MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(
460 MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
461 MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) {
462 if (!SaveSCC)
463 return PreferLast ? Last : First;
464
465 LiveRange &LR = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
466 auto MBBE = MBB.end();
467 SlotIndex FirstIdx = First != MBBE ? LIS->getInstructionIndex(*First)
468 : LIS->getMBBEndIdx(&MBB);
469 SlotIndex LastIdx =
470 Last != MBBE ? LIS->getInstructionIndex(*Last) : LIS->getMBBEndIdx(&MBB);
471 SlotIndex Idx = PreferLast ? LastIdx : FirstIdx;
472 const LiveRange::Segment *S;
473
474 for (;;) {
475 S = LR.getSegmentContaining(Idx);
476 if (!S)
477 break;
478
479 if (PreferLast) {
480 SlotIndex Next = S->start.getBaseIndex();
481 if (Next < FirstIdx)
482 break;
483 Idx = Next;
484 } else {
485 SlotIndex Next = S->end.getNextIndex().getBaseIndex();
486 if (Next > LastIdx)
487 break;
488 Idx = Next;
489 }
490 }
491
492 MachineBasicBlock::iterator MBBI;
493
494 if (MachineInstr *MI = LIS->getInstructionFromIndex(Idx))
495 MBBI = MI;
496 else {
497 assert(Idx == LIS->getMBBEndIdx(&MBB));
498 MBBI = MBB.end();
499 }
500
501 if (S)
502 MBBI = saveSCC(MBB, MBBI);
503
504 return MBBI;
505}
506
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000507void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,
508 MachineBasicBlock::iterator Before,
Nicolai Haehnlea56e6b62016-03-21 20:39:24 +0000509 unsigned SaveWQM, unsigned LiveMaskReg) {
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000510 MachineInstr *MI;
511
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000512 if (SaveWQM) {
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000513 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_SAVEEXEC_B64),
514 SaveWQM)
515 .addReg(LiveMaskReg);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000516 } else {
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000517 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_B64),
518 AMDGPU::EXEC)
519 .addReg(AMDGPU::EXEC)
520 .addReg(LiveMaskReg);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000521 }
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000522
523 LIS->InsertMachineInstrInMaps(*MI);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000524}
525
526void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB,
527 MachineBasicBlock::iterator Before,
Nicolai Haehnlea56e6b62016-03-21 20:39:24 +0000528 unsigned SavedWQM) {
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000529 MachineInstr *MI;
530
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000531 if (SavedWQM) {
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000532 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::EXEC)
533 .addReg(SavedWQM);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000534 } else {
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000535 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
536 AMDGPU::EXEC)
537 .addReg(AMDGPU::EXEC);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000538 }
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000539
540 LIS->InsertMachineInstrInMaps(*MI);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000541}
542
543void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
544 bool isEntry) {
545 auto BII = Blocks.find(&MBB);
546 if (BII == Blocks.end())
547 return;
548
549 const BlockInfo &BI = BII->second;
550
551 if (!(BI.InNeeds & StateWQM))
552 return;
553
554 // This is a non-entry block that is WQM throughout, so no need to do
555 // anything.
556 if (!isEntry && !(BI.Needs & StateExact) && BI.OutNeeds != StateExact)
557 return;
558
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000559 DEBUG(dbgs() << "\nProcessing block BB#" << MBB.getNumber() << ":\n");
560
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000561 unsigned SavedWQMReg = 0;
562 bool WQMFromExec = isEntry;
563 char State = isEntry ? StateExact : StateWQM;
564
565 auto II = MBB.getFirstNonPHI(), IE = MBB.end();
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000566 if (isEntry)
567 ++II; // Skip the instruction that saves LiveMask
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000568
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000569 MachineBasicBlock::iterator First = IE;
570 for (;;) {
571 MachineBasicBlock::iterator Next = II;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000572 char Needs = 0;
573 char OutNeeds = 0;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000574
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000575 if (First == IE)
576 First = II;
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000577
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000578 if (II != IE) {
579 MachineInstr &MI = *II;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000580
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000581 if (requiresCorrectState(MI)) {
582 auto III = Instructions.find(&MI);
583 if (III != Instructions.end()) {
584 Needs = III->second.Needs;
585 OutNeeds = III->second.OutNeeds;
586 }
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000587 }
588
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000589 if (MI.isTerminator() && !Needs && OutNeeds == StateExact)
590 Needs = StateExact;
591
592 if (MI.getOpcode() == AMDGPU::SI_ELSE && BI.OutNeeds == StateExact)
593 MI.getOperand(3).setImm(1);
594
595 ++Next;
596 } else {
597 // End of basic block
598 if (BI.OutNeeds & StateWQM)
599 Needs = StateWQM;
600 else if (BI.OutNeeds == StateExact)
601 Needs = StateExact;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000602 }
Nicolai Haehnle3b572002016-07-28 11:39:24 +0000603
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000604 if (Needs) {
605 if (Needs != State) {
606 MachineBasicBlock::iterator Before =
607 prepareInsertion(MBB, First, II, Needs == StateWQM,
608 Needs == StateExact || WQMFromExec);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000609
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000610 if (Needs == StateExact) {
611 if (!WQMFromExec && (OutNeeds & StateWQM))
612 SavedWQMReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
613
614 toExact(MBB, Before, SavedWQMReg, LiveMaskReg);
615 } else {
616 assert(WQMFromExec == (SavedWQMReg == 0));
617
618 toWQM(MBB, Before, SavedWQMReg);
619
620 if (SavedWQMReg) {
621 LIS->createAndComputeVirtRegInterval(SavedWQMReg);
622 SavedWQMReg = 0;
623 }
624 }
625
626 State = Needs;
627 }
628
629 First = IE;
630 }
631
632 if (II == IE)
633 break;
634 II = Next;
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000635 }
636}
637
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000638void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) {
639 for (MachineInstr *MI : LiveMaskQueries) {
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000640 const DebugLoc &DL = MI->getDebugLoc();
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000641 unsigned Dest = MI->getOperand(0).getReg();
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000642 MachineInstr *Copy =
643 BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest)
644 .addReg(LiveMaskReg);
645
646 LIS->ReplaceMachineInstrInMaps(*MI, *Copy);
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000647 MI->eraseFromParent();
648 }
649}
650
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000651bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
Nicolai Haehnledf3a20c2016-04-06 19:40:20 +0000652 if (MF.getFunction()->getCallingConv() != CallingConv::AMDGPU_PS)
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000653 return false;
654
655 Instructions.clear();
656 Blocks.clear();
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000657 LiveMaskQueries.clear();
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000658
Matt Arsenault43e92fe2016-06-24 06:30:11 +0000659 const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
660
661 TII = ST.getInstrInfo();
662 TRI = &TII->getRegisterInfo();
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000663 MRI = &MF.getRegInfo();
Nicolai Haehnlebef0e902016-08-02 19:17:37 +0000664 LIS = &getAnalysis<LiveIntervals>();
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000665
666 char GlobalFlags = analyzeFunction(MF);
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000667 if (!(GlobalFlags & StateWQM)) {
668 lowerLiveMaskQueries(AMDGPU::EXEC);
669 return !LiveMaskQueries.empty();
670 }
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000671
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000672 // Store a copy of the original live mask when required
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000673 unsigned LiveMaskReg = 0;
Duncan P. N. Exon Smith4d295112016-07-08 19:16:05 +0000674 {
675 MachineBasicBlock &Entry = MF.front();
676 MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI();
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000677
Duncan P. N. Exon Smith4d295112016-07-08 19:16:05 +0000678 if (GlobalFlags & StateExact || !LiveMaskQueries.empty()) {
679 LiveMaskReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000680 MachineInstr *MI = BuildMI(Entry, EntryMI, DebugLoc(),
681 TII->get(AMDGPU::COPY), LiveMaskReg)
682 .addReg(AMDGPU::EXEC);
683 LIS->InsertMachineInstrInMaps(*MI);
Duncan P. N. Exon Smith4d295112016-07-08 19:16:05 +0000684 }
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000685
Duncan P. N. Exon Smith4d295112016-07-08 19:16:05 +0000686 if (GlobalFlags == StateWQM) {
687 // For a shader that needs only WQM, we can just set it once.
688 BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
689 AMDGPU::EXEC)
690 .addReg(AMDGPU::EXEC);
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000691
Duncan P. N. Exon Smith4d295112016-07-08 19:16:05 +0000692 lowerLiveMaskQueries(LiveMaskReg);
693 // EntryMI may become invalid here
694 return true;
695 }
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000696 }
697
Nicolai Haehnle3bba6a82016-09-03 12:26:38 +0000698 DEBUG(printInfo());
699
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000700 lowerLiveMaskQueries(LiveMaskReg);
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000701
Nicolai Haehnleb0c97482016-04-22 04:04:08 +0000702 // Handle the general case
Matt Arsenault8dff86d2016-07-13 05:55:15 +0000703 for (auto BII : Blocks)
704 processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin());
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000705
Nicolai Haehnlee58e0e32016-09-12 16:25:20 +0000706 // Physical registers like SCC aren't tracked by default anyway, so just
707 // removing the ranges we computed is the simplest option for maintaining
708 // the analysis results.
709 LIS->removeRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
710
Nicolai Haehnle213e87f2016-03-21 20:28:33 +0000711 return true;
712}