blob: fae27294e364625c1f8ea9fc2a39ecf8214f5eb1 [file] [log] [blame]
Dan Gohmanee2e4032008-09-18 16:26:26 +00001//===----- ScheduleDAGFast.cpp - Fast poor list scheduler -----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements a fast scheduler.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "pre-RA-sched"
Dan Gohman84fbac52009-02-06 17:22:58 +000015#include "ScheduleDAGSDNodes.h"
Dale Johannesen6cf64a62010-08-17 22:17:24 +000016#include "llvm/InlineAsm.h"
Dan Gohmanee2e4032008-09-18 16:26:26 +000017#include "llvm/CodeGen/SchedulerRegistry.h"
Dan Gohman79ce2762009-01-15 19:20:50 +000018#include "llvm/CodeGen/SelectionDAGISel.h"
Dan Gohmanee2e4032008-09-18 16:26:26 +000019#include "llvm/Target/TargetRegisterInfo.h"
20#include "llvm/Target/TargetData.h"
Dan Gohmanee2e4032008-09-18 16:26:26 +000021#include "llvm/Target/TargetInstrInfo.h"
22#include "llvm/Support/Debug.h"
Dan Gohmanee2e4032008-09-18 16:26:26 +000023#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/STLExtras.h"
Torok Edwin7d696d82009-07-11 13:10:19 +000026#include "llvm/Support/ErrorHandling.h"
Chris Lattnerbbbfa992009-08-23 06:35:02 +000027#include "llvm/Support/raw_ostream.h"
Dan Gohmanee2e4032008-09-18 16:26:26 +000028using namespace llvm;
29
30STATISTIC(NumUnfolds, "Number of nodes unfolded");
31STATISTIC(NumDups, "Number of duplicated nodes");
Evan Chengc29a56d2009-01-12 03:19:55 +000032STATISTIC(NumPRCopies, "Number of physical copies");
Dan Gohmanee2e4032008-09-18 16:26:26 +000033
34static RegisterScheduler
Dan Gohmanb8cab922008-10-14 20:25:08 +000035 fastDAGScheduler("fast", "Fast suboptimal list scheduling",
Dan Gohmanee2e4032008-09-18 16:26:26 +000036 createFastDAGScheduler);
37
38namespace {
39 /// FastPriorityQueue - A degenerate priority queue that considers
40 /// all nodes to have the same priority.
41 ///
Nick Lewycky6726b6d2009-10-25 06:33:48 +000042 struct FastPriorityQueue {
Dan Gohman086ec992008-09-23 18:50:48 +000043 SmallVector<SUnit *, 16> Queue;
Dan Gohmanee2e4032008-09-18 16:26:26 +000044
45 bool empty() const { return Queue.empty(); }
46
47 void push(SUnit *U) {
48 Queue.push_back(U);
49 }
50
51 SUnit *pop() {
52 if (empty()) return NULL;
53 SUnit *V = Queue.back();
54 Queue.pop_back();
55 return V;
56 }
57 };
58
59//===----------------------------------------------------------------------===//
60/// ScheduleDAGFast - The actual "fast" list scheduler implementation.
61///
Nick Lewycky6726b6d2009-10-25 06:33:48 +000062class ScheduleDAGFast : public ScheduleDAGSDNodes {
Dan Gohmanee2e4032008-09-18 16:26:26 +000063private:
64 /// AvailableQueue - The priority queue to use for the available SUnits.
65 FastPriorityQueue AvailableQueue;
66
Dan Gohman086ec992008-09-23 18:50:48 +000067 /// LiveRegDefs - A set of physical registers and their definition
Dan Gohmanee2e4032008-09-18 16:26:26 +000068 /// that are "live". These nodes must be scheduled before any other nodes that
69 /// modifies the registers can be scheduled.
Dan Gohman086ec992008-09-23 18:50:48 +000070 unsigned NumLiveRegs;
Dan Gohmanee2e4032008-09-18 16:26:26 +000071 std::vector<SUnit*> LiveRegDefs;
72 std::vector<unsigned> LiveRegCycles;
73
74public:
Dan Gohman79ce2762009-01-15 19:20:50 +000075 ScheduleDAGFast(MachineFunction &mf)
76 : ScheduleDAGSDNodes(mf) {}
Dan Gohmanee2e4032008-09-18 16:26:26 +000077
78 void Schedule();
79
Dan Gohman54e4c362008-12-09 22:54:47 +000080 /// AddPred - adds a predecessor edge to SUnit SU.
Dan Gohmanee2e4032008-09-18 16:26:26 +000081 /// This returns true if this is a new predecessor.
Dan Gohmanffa39122008-12-16 01:00:55 +000082 void AddPred(SUnit *SU, const SDep &D) {
83 SU->addPred(D);
Dan Gohman54e4c362008-12-09 22:54:47 +000084 }
Dan Gohmanee2e4032008-09-18 16:26:26 +000085
Dan Gohman54e4c362008-12-09 22:54:47 +000086 /// RemovePred - removes a predecessor edge from SUnit SU.
87 /// This returns true if an edge was removed.
Dan Gohmanffa39122008-12-16 01:00:55 +000088 void RemovePred(SUnit *SU, const SDep &D) {
89 SU->removePred(D);
Dan Gohman54e4c362008-12-09 22:54:47 +000090 }
Dan Gohmanee2e4032008-09-18 16:26:26 +000091
92private:
Dan Gohman54e4c362008-12-09 22:54:47 +000093 void ReleasePred(SUnit *SU, SDep *PredEdge);
Dan Gohman9e64bbb2009-02-10 23:27:53 +000094 void ReleasePredecessors(SUnit *SU, unsigned CurCycle);
Dan Gohmanee2e4032008-09-18 16:26:26 +000095 void ScheduleNodeBottomUp(SUnit*, unsigned);
96 SUnit *CopyAndMoveSuccessors(SUnit*);
Evan Chengc29a56d2009-01-12 03:19:55 +000097 void InsertCopiesAndMoveSuccs(SUnit*, unsigned,
98 const TargetRegisterClass*,
99 const TargetRegisterClass*,
100 SmallVector<SUnit*, 2>&);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000101 bool DelayForLiveRegsBottomUp(SUnit*, SmallVector<unsigned, 4>&);
102 void ListScheduleBottomUp();
Dan Gohman3f237442008-12-16 03:25:46 +0000103
104 /// ForceUnitLatencies - The fast scheduler doesn't care about real latencies.
105 bool ForceUnitLatencies() const { return true; }
Dan Gohmanee2e4032008-09-18 16:26:26 +0000106};
107} // end anonymous namespace
108
109
110/// Schedule - Schedule the DAG using list scheduling.
111void ScheduleDAGFast::Schedule() {
David Greene33db62c2010-01-05 01:25:09 +0000112 DEBUG(dbgs() << "********** List Scheduling **********\n");
Dan Gohmanee2e4032008-09-18 16:26:26 +0000113
Dan Gohman086ec992008-09-23 18:50:48 +0000114 NumLiveRegs = 0;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000115 LiveRegDefs.resize(TRI->getNumRegs(), NULL);
116 LiveRegCycles.resize(TRI->getNumRegs(), 0);
117
Dan Gohmanc9a5b9e2008-12-23 18:36:58 +0000118 // Build the scheduling graph.
Dan Gohman98976e42009-10-09 23:33:48 +0000119 BuildSchedGraph(NULL);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000120
121 DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
Dan Gohman3cc62432008-11-18 02:06:40 +0000122 SUnits[su].dumpAll(this));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000123
124 // Execute the actual scheduling loop.
125 ListScheduleBottomUp();
126}
127
128//===----------------------------------------------------------------------===//
129// Bottom-Up Scheduling
130//===----------------------------------------------------------------------===//
131
132/// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to
133/// the AvailableQueue if the count reaches zero. Also update its cycle bound.
Dan Gohman54e4c362008-12-09 22:54:47 +0000134void ScheduleDAGFast::ReleasePred(SUnit *SU, SDep *PredEdge) {
135 SUnit *PredSU = PredEdge->getSUnit();
Reid Klecknerc277ab02009-09-30 20:15:38 +0000136
Dan Gohmanee2e4032008-09-18 16:26:26 +0000137#ifndef NDEBUG
Reid Klecknerc277ab02009-09-30 20:15:38 +0000138 if (PredSU->NumSuccsLeft == 0) {
David Greene33db62c2010-01-05 01:25:09 +0000139 dbgs() << "*** Scheduling failed! ***\n";
Dan Gohman3cc62432008-11-18 02:06:40 +0000140 PredSU->dump(this);
David Greene33db62c2010-01-05 01:25:09 +0000141 dbgs() << " has been released too many times!\n";
Torok Edwinc23197a2009-07-14 16:55:14 +0000142 llvm_unreachable(0);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000143 }
144#endif
Reid Klecknerc277ab02009-09-30 20:15:38 +0000145 --PredSU->NumSuccsLeft;
146
Dan Gohman9e64bbb2009-02-10 23:27:53 +0000147 // If all the node's successors are scheduled, this node is ready
148 // to be scheduled. Ignore the special EntrySU node.
149 if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) {
Dan Gohmanee2e4032008-09-18 16:26:26 +0000150 PredSU->isAvailable = true;
151 AvailableQueue.push(PredSU);
152 }
153}
154
Dan Gohman9e64bbb2009-02-10 23:27:53 +0000155void ScheduleDAGFast::ReleasePredecessors(SUnit *SU, unsigned CurCycle) {
Dan Gohmanee2e4032008-09-18 16:26:26 +0000156 // Bottom up: release predecessors
157 for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
158 I != E; ++I) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000159 ReleasePred(SU, &*I);
160 if (I->isAssignedRegDep()) {
Dan Gohmanee2e4032008-09-18 16:26:26 +0000161 // This is a physical register dependency and it's impossible or
162 // expensive to copy the register. Make sure nothing that can
163 // clobber the register is scheduled between the predecessor and
164 // this node.
Dan Gohman54e4c362008-12-09 22:54:47 +0000165 if (!LiveRegDefs[I->getReg()]) {
Dan Gohman086ec992008-09-23 18:50:48 +0000166 ++NumLiveRegs;
Dan Gohman54e4c362008-12-09 22:54:47 +0000167 LiveRegDefs[I->getReg()] = I->getSUnit();
168 LiveRegCycles[I->getReg()] = CurCycle;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000169 }
170 }
171 }
Dan Gohman9e64bbb2009-02-10 23:27:53 +0000172}
173
174/// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
175/// count of its predecessors. If a predecessor pending count is zero, add it to
176/// the Available queue.
177void ScheduleDAGFast::ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle) {
David Greene33db62c2010-01-05 01:25:09 +0000178 DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: ");
Dan Gohman9e64bbb2009-02-10 23:27:53 +0000179 DEBUG(SU->dump(this));
180
181 assert(CurCycle >= SU->getHeight() && "Node scheduled below its height!");
182 SU->setHeightToAtLeast(CurCycle);
183 Sequence.push_back(SU);
184
185 ReleasePredecessors(SU, CurCycle);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000186
187 // Release all the implicit physical register defs that are live.
188 for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
189 I != E; ++I) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000190 if (I->isAssignedRegDep()) {
Dan Gohman3f237442008-12-16 03:25:46 +0000191 if (LiveRegCycles[I->getReg()] == I->getSUnit()->getHeight()) {
Dan Gohman086ec992008-09-23 18:50:48 +0000192 assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
Dan Gohman54e4c362008-12-09 22:54:47 +0000193 assert(LiveRegDefs[I->getReg()] == SU &&
Dan Gohmanee2e4032008-09-18 16:26:26 +0000194 "Physical register dependency violated?");
Dan Gohman086ec992008-09-23 18:50:48 +0000195 --NumLiveRegs;
Dan Gohman54e4c362008-12-09 22:54:47 +0000196 LiveRegDefs[I->getReg()] = NULL;
197 LiveRegCycles[I->getReg()] = 0;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000198 }
199 }
200 }
201
202 SU->isScheduled = true;
203}
204
Dan Gohmanee2e4032008-09-18 16:26:26 +0000205/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled
206/// successors to the newly created node.
207SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
Dan Gohmand23e0f82008-11-13 23:24:17 +0000208 if (SU->getNode()->getFlaggedNode())
Dan Gohmanee2e4032008-09-18 16:26:26 +0000209 return NULL;
210
Dan Gohman550f5af2008-11-13 21:36:12 +0000211 SDNode *N = SU->getNode();
Dan Gohmanee2e4032008-09-18 16:26:26 +0000212 if (!N)
213 return NULL;
214
215 SUnit *NewSU;
216 bool TryUnfold = false;
217 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
Owen Andersone50ed302009-08-10 22:56:29 +0000218 EVT VT = N->getValueType(i);
Owen Anderson825b72b2009-08-11 20:47:22 +0000219 if (VT == MVT::Flag)
Dan Gohmanee2e4032008-09-18 16:26:26 +0000220 return NULL;
Owen Anderson825b72b2009-08-11 20:47:22 +0000221 else if (VT == MVT::Other)
Dan Gohmanee2e4032008-09-18 16:26:26 +0000222 TryUnfold = true;
223 }
224 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
225 const SDValue &Op = N->getOperand(i);
Owen Andersone50ed302009-08-10 22:56:29 +0000226 EVT VT = Op.getNode()->getValueType(Op.getResNo());
Owen Anderson825b72b2009-08-11 20:47:22 +0000227 if (VT == MVT::Flag)
Dan Gohmanee2e4032008-09-18 16:26:26 +0000228 return NULL;
229 }
230
231 if (TryUnfold) {
232 SmallVector<SDNode*, 2> NewNodes;
Dan Gohmana23b3b82008-11-13 21:21:28 +0000233 if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes))
Dan Gohmanee2e4032008-09-18 16:26:26 +0000234 return NULL;
235
David Greene33db62c2010-01-05 01:25:09 +0000236 DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n");
Dan Gohmanee2e4032008-09-18 16:26:26 +0000237 assert(NewNodes.size() == 2 && "Expected a load folding node!");
238
239 N = NewNodes[1];
240 SDNode *LoadNode = NewNodes[0];
241 unsigned NumVals = N->getNumValues();
Dan Gohman550f5af2008-11-13 21:36:12 +0000242 unsigned OldNumVals = SU->getNode()->getNumValues();
Dan Gohmanee2e4032008-09-18 16:26:26 +0000243 for (unsigned i = 0; i != NumVals; ++i)
Dan Gohman550f5af2008-11-13 21:36:12 +0000244 DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
245 DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1),
Dan Gohmana23b3b82008-11-13 21:21:28 +0000246 SDValue(LoadNode, 1));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000247
Dan Gohmancdb260d2008-11-19 23:39:02 +0000248 SUnit *NewSU = NewSUnit(N);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000249 assert(N->getNodeId() == -1 && "Node already inserted!");
250 N->setNodeId(NewSU->NodeNum);
251
252 const TargetInstrDesc &TID = TII->get(N->getMachineOpcode());
253 for (unsigned i = 0; i != TID.getNumOperands(); ++i) {
254 if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) {
255 NewSU->isTwoAddress = true;
256 break;
257 }
258 }
259 if (TID.isCommutable())
260 NewSU->isCommutable = true;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000261
262 // LoadNode may already exist. This can happen when there is another
263 // load from the same location and producing the same type of value
264 // but it has different alignment or volatileness.
265 bool isNewLoad = true;
266 SUnit *LoadSU;
267 if (LoadNode->getNodeId() != -1) {
268 LoadSU = &SUnits[LoadNode->getNodeId()];
269 isNewLoad = false;
270 } else {
Dan Gohmancdb260d2008-11-19 23:39:02 +0000271 LoadSU = NewSUnit(LoadNode);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000272 LoadNode->setNodeId(LoadSU->NodeNum);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000273 }
274
Dan Gohman54e4c362008-12-09 22:54:47 +0000275 SDep ChainPred;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000276 SmallVector<SDep, 4> ChainSuccs;
277 SmallVector<SDep, 4> LoadPreds;
278 SmallVector<SDep, 4> NodePreds;
279 SmallVector<SDep, 4> NodeSuccs;
280 for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
281 I != E; ++I) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000282 if (I->isCtrl())
283 ChainPred = *I;
284 else if (I->getSUnit()->getNode() &&
285 I->getSUnit()->getNode()->isOperandOf(LoadNode))
286 LoadPreds.push_back(*I);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000287 else
Dan Gohman54e4c362008-12-09 22:54:47 +0000288 NodePreds.push_back(*I);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000289 }
290 for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
291 I != E; ++I) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000292 if (I->isCtrl())
293 ChainSuccs.push_back(*I);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000294 else
Dan Gohman54e4c362008-12-09 22:54:47 +0000295 NodeSuccs.push_back(*I);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000296 }
297
Dan Gohman54e4c362008-12-09 22:54:47 +0000298 if (ChainPred.getSUnit()) {
299 RemovePred(SU, ChainPred);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000300 if (isNewLoad)
Dan Gohman54e4c362008-12-09 22:54:47 +0000301 AddPred(LoadSU, ChainPred);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000302 }
303 for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000304 const SDep &Pred = LoadPreds[i];
305 RemovePred(SU, Pred);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000306 if (isNewLoad) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000307 AddPred(LoadSU, Pred);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000308 }
309 }
310 for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000311 const SDep &Pred = NodePreds[i];
312 RemovePred(SU, Pred);
313 AddPred(NewSU, Pred);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000314 }
315 for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000316 SDep D = NodeSuccs[i];
317 SUnit *SuccDep = D.getSUnit();
318 D.setSUnit(SU);
319 RemovePred(SuccDep, D);
320 D.setSUnit(NewSU);
321 AddPred(SuccDep, D);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000322 }
323 for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000324 SDep D = ChainSuccs[i];
325 SUnit *SuccDep = D.getSUnit();
326 D.setSUnit(SU);
327 RemovePred(SuccDep, D);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000328 if (isNewLoad) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000329 D.setSUnit(LoadSU);
330 AddPred(SuccDep, D);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000331 }
332 }
333 if (isNewLoad) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000334 AddPred(NewSU, SDep(LoadSU, SDep::Order, LoadSU->Latency));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000335 }
336
337 ++NumUnfolds;
338
339 if (NewSU->NumSuccsLeft == 0) {
340 NewSU->isAvailable = true;
341 return NewSU;
342 }
343 SU = NewSU;
344 }
345
David Greene33db62c2010-01-05 01:25:09 +0000346 DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n");
Dan Gohmancdb260d2008-11-19 23:39:02 +0000347 NewSU = Clone(SU);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000348
349 // New SUnit has the exact same predecessors.
350 for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
351 I != E; ++I)
Dan Gohman3f237442008-12-16 03:25:46 +0000352 if (!I->isArtificial())
Dan Gohman54e4c362008-12-09 22:54:47 +0000353 AddPred(NewSU, *I);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000354
355 // Only copy scheduled successors. Cut them from old node's successor
356 // list and move them over.
Dan Gohman54e4c362008-12-09 22:54:47 +0000357 SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000358 for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
359 I != E; ++I) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000360 if (I->isArtificial())
Dan Gohmanee2e4032008-09-18 16:26:26 +0000361 continue;
Dan Gohman54e4c362008-12-09 22:54:47 +0000362 SUnit *SuccSU = I->getSUnit();
363 if (SuccSU->isScheduled) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000364 SDep D = *I;
365 D.setSUnit(NewSU);
366 AddPred(SuccSU, D);
367 D.setSUnit(SU);
368 DelDeps.push_back(std::make_pair(SuccSU, D));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000369 }
370 }
Evan Chengc29a56d2009-01-12 03:19:55 +0000371 for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
Dan Gohman54e4c362008-12-09 22:54:47 +0000372 RemovePred(DelDeps[i].first, DelDeps[i].second);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000373
374 ++NumDups;
375 return NewSU;
376}
377
Evan Chengc29a56d2009-01-12 03:19:55 +0000378/// InsertCopiesAndMoveSuccs - Insert register copies and move all
379/// scheduled successors of the given SUnit to the last copy.
380void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
Dan Gohmanee2e4032008-09-18 16:26:26 +0000381 const TargetRegisterClass *DestRC,
382 const TargetRegisterClass *SrcRC,
383 SmallVector<SUnit*, 2> &Copies) {
Dan Gohmancdb260d2008-11-19 23:39:02 +0000384 SUnit *CopyFromSU = NewSUnit(static_cast<SDNode *>(NULL));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000385 CopyFromSU->CopySrcRC = SrcRC;
386 CopyFromSU->CopyDstRC = DestRC;
387
Dan Gohmancdb260d2008-11-19 23:39:02 +0000388 SUnit *CopyToSU = NewSUnit(static_cast<SDNode *>(NULL));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000389 CopyToSU->CopySrcRC = DestRC;
390 CopyToSU->CopyDstRC = SrcRC;
391
392 // Only copy scheduled successors. Cut them from old node's successor
393 // list and move them over.
Dan Gohman54e4c362008-12-09 22:54:47 +0000394 SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000395 for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
396 I != E; ++I) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000397 if (I->isArtificial())
Dan Gohmanee2e4032008-09-18 16:26:26 +0000398 continue;
Dan Gohman54e4c362008-12-09 22:54:47 +0000399 SUnit *SuccSU = I->getSUnit();
400 if (SuccSU->isScheduled) {
401 SDep D = *I;
402 D.setSUnit(CopyToSU);
403 AddPred(SuccSU, D);
404 DelDeps.push_back(std::make_pair(SuccSU, *I));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000405 }
406 }
407 for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000408 RemovePred(DelDeps[i].first, DelDeps[i].second);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000409 }
410
Dan Gohman54e4c362008-12-09 22:54:47 +0000411 AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg));
412 AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000413
414 Copies.push_back(CopyFromSU);
415 Copies.push_back(CopyToSU);
416
Evan Chengc29a56d2009-01-12 03:19:55 +0000417 ++NumPRCopies;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000418}
419
420/// getPhysicalRegisterVT - Returns the ValueType of the physical register
421/// definition of the specified node.
422/// FIXME: Move to SelectionDAG?
Owen Andersone50ed302009-08-10 22:56:29 +0000423static EVT getPhysicalRegisterVT(SDNode *N, unsigned Reg,
Dan Gohmanee2e4032008-09-18 16:26:26 +0000424 const TargetInstrInfo *TII) {
425 const TargetInstrDesc &TID = TII->get(N->getMachineOpcode());
426 assert(TID.ImplicitDefs && "Physical reg def must be in implicit def list!");
427 unsigned NumRes = TID.getNumDefs();
428 for (const unsigned *ImpDef = TID.getImplicitDefs(); *ImpDef; ++ImpDef) {
429 if (Reg == *ImpDef)
430 break;
431 ++NumRes;
432 }
433 return N->getValueType(NumRes);
434}
435
Dale Johannesen6cf64a62010-08-17 22:17:24 +0000436/// CheckForLiveRegDef - Return true and update live register vector if the
437/// specified register def of the specified SUnit clobbers any "live" registers.
438static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg,
439 std::vector<SUnit*> &LiveRegDefs,
440 SmallSet<unsigned, 4> &RegAdded,
441 SmallVector<unsigned, 4> &LRegs,
442 const TargetRegisterInfo *TRI) {
443 bool Added = false;
444 if (LiveRegDefs[Reg] && LiveRegDefs[Reg] != SU) {
445 if (RegAdded.insert(Reg)) {
446 LRegs.push_back(Reg);
447 Added = true;
448 }
449 }
450 for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias)
451 if (LiveRegDefs[*Alias] && LiveRegDefs[*Alias] != SU) {
452 if (RegAdded.insert(*Alias)) {
453 LRegs.push_back(*Alias);
454 Added = true;
455 }
456 }
457 return Added;
458}
459
Dan Gohmanee2e4032008-09-18 16:26:26 +0000460/// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay
461/// scheduling of the given node to satisfy live physical register dependencies.
462/// If the specific node is the last one that's available to schedule, do
463/// whatever is necessary (i.e. backtracking or cloning) to make it possible.
464bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU,
465 SmallVector<unsigned, 4> &LRegs){
Dan Gohman086ec992008-09-23 18:50:48 +0000466 if (NumLiveRegs == 0)
Dan Gohmanee2e4032008-09-18 16:26:26 +0000467 return false;
468
469 SmallSet<unsigned, 4> RegAdded;
470 // If this node would clobber any "live" register, then it's not ready.
471 for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
472 I != E; ++I) {
Dan Gohman54e4c362008-12-09 22:54:47 +0000473 if (I->isAssignedRegDep()) {
Dale Johannesen6cf64a62010-08-17 22:17:24 +0000474 CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs,
475 RegAdded, LRegs, TRI);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000476 }
477 }
478
Dan Gohmand23e0f82008-11-13 23:24:17 +0000479 for (SDNode *Node = SU->getNode(); Node; Node = Node->getFlaggedNode()) {
Dale Johannesen6cf64a62010-08-17 22:17:24 +0000480 if (Node->getOpcode() == ISD::INLINEASM) {
481 // Inline asm can clobber physical defs.
482 unsigned NumOps = Node->getNumOperands();
483 if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag)
484 --NumOps; // Ignore the flag operand.
485
486 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
487 unsigned Flags =
488 cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
489 unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
490
491 ++i; // Skip the ID value.
492 if (InlineAsm::isRegDefKind(Flags) ||
493 InlineAsm::isRegDefEarlyClobberKind(Flags)) {
494 // Check for def of register or earlyclobber register.
495 for (; NumVals; --NumVals, ++i) {
496 unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
497 if (TargetRegisterInfo::isPhysicalRegister(Reg))
498 CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI);
499 }
500 } else
501 i += NumVals;
502 }
503 continue;
504 }
Dan Gohmand23e0f82008-11-13 23:24:17 +0000505 if (!Node->isMachineOpcode())
Dan Gohmanee2e4032008-09-18 16:26:26 +0000506 continue;
507 const TargetInstrDesc &TID = TII->get(Node->getMachineOpcode());
508 if (!TID.ImplicitDefs)
509 continue;
510 for (const unsigned *Reg = TID.ImplicitDefs; *Reg; ++Reg) {
Dale Johannesen6cf64a62010-08-17 22:17:24 +0000511 CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000512 }
513 }
514 return !LRegs.empty();
515}
516
517
518/// ListScheduleBottomUp - The main loop of list scheduling for bottom-up
519/// schedulers.
520void ScheduleDAGFast::ListScheduleBottomUp() {
521 unsigned CurCycle = 0;
Dan Gohman9e64bbb2009-02-10 23:27:53 +0000522
523 // Release any predecessors of the special Exit node.
524 ReleasePredecessors(&ExitSU, CurCycle);
525
Dan Gohmanee2e4032008-09-18 16:26:26 +0000526 // Add root to Available queue.
527 if (!SUnits.empty()) {
Dan Gohmana23b3b82008-11-13 21:21:28 +0000528 SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
Dan Gohmanee2e4032008-09-18 16:26:26 +0000529 assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!");
530 RootSU->isAvailable = true;
531 AvailableQueue.push(RootSU);
532 }
533
534 // While Available queue is not empty, grab the node with the highest
535 // priority. If it is not ready put it back. Schedule the node.
536 SmallVector<SUnit*, 4> NotReady;
537 DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMap;
538 Sequence.reserve(SUnits.size());
539 while (!AvailableQueue.empty()) {
540 bool Delayed = false;
541 LRegsMap.clear();
542 SUnit *CurSU = AvailableQueue.pop();
543 while (CurSU) {
Dan Gohmane93483d2008-11-17 19:52:36 +0000544 SmallVector<unsigned, 4> LRegs;
545 if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
546 break;
547 Delayed = true;
548 LRegsMap.insert(std::make_pair(CurSU, LRegs));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000549
550 CurSU->isPending = true; // This SU is not in AvailableQueue right now.
551 NotReady.push_back(CurSU);
552 CurSU = AvailableQueue.pop();
553 }
554
555 // All candidates are delayed due to live physical reg dependencies.
556 // Try code duplication or inserting cross class copies
557 // to resolve it.
558 if (Delayed && !CurSU) {
559 if (!CurSU) {
560 // Try duplicating the nodes that produces these
561 // "expensive to copy" values to break the dependency. In case even
562 // that doesn't work, insert cross class copies.
563 SUnit *TrySU = NotReady[0];
564 SmallVector<unsigned, 4> &LRegs = LRegsMap[TrySU];
565 assert(LRegs.size() == 1 && "Can't handle this yet!");
566 unsigned Reg = LRegs[0];
567 SUnit *LRDef = LiveRegDefs[Reg];
Owen Andersone50ed302009-08-10 22:56:29 +0000568 EVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII);
Evan Chengc29a56d2009-01-12 03:19:55 +0000569 const TargetRegisterClass *RC =
Rafael Espindolad31f9722010-06-29 14:02:34 +0000570 TRI->getMinimalPhysRegClass(Reg, VT);
Evan Chengc29a56d2009-01-12 03:19:55 +0000571 const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC);
572
573 // If cross copy register class is null, then it must be possible copy
574 // the value directly. Do not try duplicate the def.
575 SUnit *NewDef = 0;
576 if (DestRC)
577 NewDef = CopyAndMoveSuccessors(LRDef);
578 else
579 DestRC = RC;
Dan Gohmanee2e4032008-09-18 16:26:26 +0000580 if (!NewDef) {
Evan Chengc29a56d2009-01-12 03:19:55 +0000581 // Issue copies, these can be expensive cross register class copies.
Dan Gohmanee2e4032008-09-18 16:26:26 +0000582 SmallVector<SUnit*, 2> Copies;
Evan Chengc29a56d2009-01-12 03:19:55 +0000583 InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
David Greene33db62c2010-01-05 01:25:09 +0000584 DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000585 << " to SU #" << Copies.front()->NodeNum << "\n");
Dan Gohman54e4c362008-12-09 22:54:47 +0000586 AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1,
587 /*Reg=*/0, /*isNormalMemory=*/false,
588 /*isMustAlias=*/false, /*isArtificial=*/true));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000589 NewDef = Copies.back();
590 }
591
David Greene33db62c2010-01-05 01:25:09 +0000592 DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
Chris Lattnerbbbfa992009-08-23 06:35:02 +0000593 << " to SU #" << TrySU->NodeNum << "\n");
Dan Gohmanee2e4032008-09-18 16:26:26 +0000594 LiveRegDefs[Reg] = NewDef;
Dan Gohman54e4c362008-12-09 22:54:47 +0000595 AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1,
596 /*Reg=*/0, /*isNormalMemory=*/false,
597 /*isMustAlias=*/false, /*isArtificial=*/true));
Dan Gohmanee2e4032008-09-18 16:26:26 +0000598 TrySU->isAvailable = false;
599 CurSU = NewDef;
600 }
601
602 if (!CurSU) {
Torok Edwinc23197a2009-07-14 16:55:14 +0000603 llvm_unreachable("Unable to resolve live physical register dependencies!");
Dan Gohmanee2e4032008-09-18 16:26:26 +0000604 }
605 }
606
607 // Add the nodes that aren't ready back onto the available list.
608 for (unsigned i = 0, e = NotReady.size(); i != e; ++i) {
609 NotReady[i]->isPending = false;
610 // May no longer be available due to backtracking.
611 if (NotReady[i]->isAvailable)
612 AvailableQueue.push(NotReady[i]);
613 }
614 NotReady.clear();
615
Dan Gohman47d1a212008-11-21 00:10:42 +0000616 if (CurSU)
Dan Gohmanee2e4032008-09-18 16:26:26 +0000617 ScheduleNodeBottomUp(CurSU, CurCycle);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000618 ++CurCycle;
619 }
620
Dan Gohman937d2d82009-09-28 16:09:41 +0000621 // Reverse the order since it is bottom up.
Dan Gohmanee2e4032008-09-18 16:26:26 +0000622 std::reverse(Sequence.begin(), Sequence.end());
Dan Gohman937d2d82009-09-28 16:09:41 +0000623
Dan Gohmanee2e4032008-09-18 16:26:26 +0000624#ifndef NDEBUG
Dan Gohman937d2d82009-09-28 16:09:41 +0000625 VerifySchedule(/*isBottomUp=*/true);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000626#endif
627}
628
629//===----------------------------------------------------------------------===//
630// Public Constructor Functions
631//===----------------------------------------------------------------------===//
632
Dan Gohman47ac0f02009-02-11 04:27:20 +0000633llvm::ScheduleDAGSDNodes *
Bill Wendling98a366d2009-04-29 23:29:43 +0000634llvm::createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
Dan Gohman79ce2762009-01-15 19:20:50 +0000635 return new ScheduleDAGFast(*IS->MF);
Dan Gohmanee2e4032008-09-18 16:26:26 +0000636}