blob: 702d21228477aeaa457e789e2b4dfec48174d25d [file] [log] [blame]
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +00001//===---- MachineCombiner.cpp - Instcombining on SSA form machine code ----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// The machine combiner pass uses machine trace metrics to ensure the combined
Eric Christopher17ce8a22017-03-15 21:50:46 +000011// instructions do not lengthen the critical path or the resource depth.
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000012//===----------------------------------------------------------------------===//
Hans Wennborg083ca9b2015-10-06 23:24:35 +000013
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000014#include "llvm/ADT/DenseMap.h"
Mehdi Aminib550cb12016-04-18 09:17:29 +000015#include "llvm/ADT/Statistic.h"
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000016#include "llvm/CodeGen/MachineDominators.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineFunctionPass.h"
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000019#include "llvm/CodeGen/MachineLoopInfo.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/MachineTraceMetrics.h"
22#include "llvm/CodeGen/Passes.h"
David Blaikie3f833ed2017-11-08 01:01:31 +000023#include "llvm/CodeGen/TargetInstrInfo.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000024#include "llvm/CodeGen/TargetRegisterInfo.h"
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000025#include "llvm/CodeGen/TargetSchedule.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000026#include "llvm/CodeGen/TargetSubtargetInfo.h"
Florian Hahnceb44942017-09-20 11:54:37 +000027#include "llvm/Support/CommandLine.h"
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000028#include "llvm/Support/Debug.h"
29#include "llvm/Support/raw_ostream.h"
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000030
31using namespace llvm;
32
Jakub Kuderski1d2dc682017-07-13 19:30:52 +000033#define DEBUG_TYPE "machine-combiner"
34
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000035STATISTIC(NumInstCombined, "Number of machineinst combined");
36
Florian Hahnceb44942017-09-20 11:54:37 +000037static cl::opt<unsigned>
38inc_threshold("machine-combiner-inc-threshold", cl::Hidden,
39 cl::desc("Incremental depth computation will be used for basic "
40 "blocks with more instructions."), cl::init(500));
41
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000042namespace {
43class MachineCombiner : public MachineFunctionPass {
44 const TargetInstrInfo *TII;
45 const TargetRegisterInfo *TRI;
Pete Cooper11759452014-09-02 17:43:54 +000046 MCSchedModel SchedModel;
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000047 MachineRegisterInfo *MRI;
Gerolf Hoflehner01b3a6182016-04-24 05:14:01 +000048 MachineLoopInfo *MLI; // Current MachineLoopInfo
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000049 MachineTraceMetrics *Traces;
50 MachineTraceMetrics::Ensemble *MinInstr;
51
52 TargetSchedModel TSchedModel;
53
Sanjay Patelb1ca4e42015-01-27 22:26:56 +000054 /// True if optimizing for code size.
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000055 bool OptSize;
56
57public:
58 static char ID;
59 MachineCombiner() : MachineFunctionPass(ID) {
60 initializeMachineCombinerPass(*PassRegistry::getPassRegistry());
61 }
62 void getAnalysisUsage(AnalysisUsage &AU) const override;
63 bool runOnMachineFunction(MachineFunction &MF) override;
Mehdi Amini117296c2016-10-01 02:56:57 +000064 StringRef getPassName() const override { return "Machine InstCombiner"; }
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000065
66private:
67 bool doSubstitute(unsigned NewSize, unsigned OldSize);
68 bool combineInstructions(MachineBasicBlock *);
69 MachineInstr *getOperandDef(const MachineOperand &MO);
70 unsigned getDepth(SmallVectorImpl<MachineInstr *> &InsInstrs,
71 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
72 MachineTraceMetrics::Trace BlockTrace);
73 unsigned getLatency(MachineInstr *Root, MachineInstr *NewRoot,
74 MachineTraceMetrics::Trace BlockTrace);
75 bool
Sanjay Patele79b43a2015-06-23 00:39:40 +000076 improvesCriticalPathLen(MachineBasicBlock *MBB, MachineInstr *Root,
Sanjay Patel766589e2015-11-10 16:48:53 +000077 MachineTraceMetrics::Trace BlockTrace,
78 SmallVectorImpl<MachineInstr *> &InsInstrs,
Sebastian Pope08d9c72016-12-11 19:39:32 +000079 SmallVectorImpl<MachineInstr *> &DelInstrs,
Sanjay Patel766589e2015-11-10 16:48:53 +000080 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
Florian Hahnceb44942017-09-20 11:54:37 +000081 MachineCombinerPattern Pattern, bool SlackIsAccurate);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000082 bool preservesResourceLen(MachineBasicBlock *MBB,
83 MachineTraceMetrics::Trace BlockTrace,
84 SmallVectorImpl<MachineInstr *> &InsInstrs,
85 SmallVectorImpl<MachineInstr *> &DelInstrs);
86 void instr2instrSC(SmallVectorImpl<MachineInstr *> &Instrs,
87 SmallVectorImpl<const MCSchedClassDesc *> &InstrsSC);
88};
Alexander Kornienkof00654e2015-06-23 09:49:53 +000089}
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000090
91char MachineCombiner::ID = 0;
92char &llvm::MachineCombinerID = MachineCombiner::ID;
93
Matthias Braun1527baa2017-05-25 21:26:32 +000094INITIALIZE_PASS_BEGIN(MachineCombiner, DEBUG_TYPE,
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000095 "Machine InstCombiner", false, false)
Gerolf Hoflehner01b3a6182016-04-24 05:14:01 +000096INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000097INITIALIZE_PASS_DEPENDENCY(MachineTraceMetrics)
Matthias Braun1527baa2017-05-25 21:26:32 +000098INITIALIZE_PASS_END(MachineCombiner, DEBUG_TYPE, "Machine InstCombiner",
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +000099 false, false)
100
101void MachineCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
102 AU.setPreservesCFG();
103 AU.addPreserved<MachineDominatorTree>();
Gerolf Hoflehner01b3a6182016-04-24 05:14:01 +0000104 AU.addRequired<MachineLoopInfo>();
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000105 AU.addPreserved<MachineLoopInfo>();
106 AU.addRequired<MachineTraceMetrics>();
107 AU.addPreserved<MachineTraceMetrics>();
108 MachineFunctionPass::getAnalysisUsage(AU);
109}
110
111MachineInstr *MachineCombiner::getOperandDef(const MachineOperand &MO) {
112 MachineInstr *DefInstr = nullptr;
113 // We need a virtual register definition.
114 if (MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg()))
115 DefInstr = MRI->getUniqueVRegDef(MO.getReg());
116 // PHI's have no depth etc.
117 if (DefInstr && DefInstr->isPHI())
118 DefInstr = nullptr;
119 return DefInstr;
120}
121
Sanjay Patelb1ca4e42015-01-27 22:26:56 +0000122/// Computes depth of instructions in vector \InsInstr.
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000123///
124/// \param InsInstrs is a vector of machine instructions
125/// \param InstrIdxForVirtReg is a dense map of virtual register to index
126/// of defining machine instruction in \p InsInstrs
127/// \param BlockTrace is a trace of machine instructions
128///
129/// \returns Depth of last instruction in \InsInstrs ("NewRoot")
130unsigned
131MachineCombiner::getDepth(SmallVectorImpl<MachineInstr *> &InsInstrs,
132 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
133 MachineTraceMetrics::Trace BlockTrace) {
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000134 SmallVector<unsigned, 16> InstrDepth;
Hal Finkele0fa8f22015-07-15 08:22:23 +0000135 assert(TSchedModel.hasInstrSchedModelOrItineraries() &&
136 "Missing machine model\n");
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000137
Sanjay Patel6b280772015-01-27 22:16:52 +0000138 // For each instruction in the new sequence compute the depth based on the
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000139 // operands. Use the trace information when possible. For new operands which
140 // are tracked in the InstrIdxForVirtReg map depth is looked up in InstrDepth
141 for (auto *InstrPtr : InsInstrs) { // for each Use
142 unsigned IDepth = 0;
Matthias Brauna4976c62017-01-29 18:20:42 +0000143 DEBUG(dbgs() << "NEW INSTR ";
144 InstrPtr->print(dbgs(), TII);
145 dbgs() << "\n";);
Sanjay Patelf69f4e42015-05-21 17:43:26 +0000146 for (const MachineOperand &MO : InstrPtr->operands()) {
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000147 // Check for virtual register operand.
148 if (!(MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg())))
149 continue;
150 if (!MO.isUse())
151 continue;
152 unsigned DepthOp = 0;
153 unsigned LatencyOp = 0;
154 DenseMap<unsigned, unsigned>::iterator II =
155 InstrIdxForVirtReg.find(MO.getReg());
156 if (II != InstrIdxForVirtReg.end()) {
157 // Operand is new virtual register not in trace
Saleem Abdulrasoolbefa2152014-08-03 23:00:38 +0000158 assert(II->second < InstrDepth.size() && "Bad Index");
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000159 MachineInstr *DefInstr = InsInstrs[II->second];
160 assert(DefInstr &&
161 "There must be a definition for a new virtual register");
162 DepthOp = InstrDepth[II->second];
Simon Pilgrim194693e2017-10-30 17:24:40 +0000163 int DefIdx = DefInstr->findRegisterDefOperandIdx(MO.getReg());
164 int UseIdx = InstrPtr->findRegisterUseOperandIdx(MO.getReg());
165 LatencyOp = TSchedModel.computeOperandLatency(DefInstr, DefIdx,
166 InstrPtr, UseIdx);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000167 } else {
168 MachineInstr *DefInstr = getOperandDef(MO);
169 if (DefInstr) {
Duncan P. N. Exon Smithe59c8af2016-02-22 03:33:28 +0000170 DepthOp = BlockTrace.getInstrCycles(*DefInstr).Depth;
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000171 LatencyOp = TSchedModel.computeOperandLatency(
172 DefInstr, DefInstr->findRegisterDefOperandIdx(MO.getReg()),
173 InstrPtr, InstrPtr->findRegisterUseOperandIdx(MO.getReg()));
174 }
175 }
176 IDepth = std::max(IDepth, DepthOp + LatencyOp);
177 }
178 InstrDepth.push_back(IDepth);
179 }
180 unsigned NewRootIdx = InsInstrs.size() - 1;
181 return InstrDepth[NewRootIdx];
182}
183
Sanjay Patelb1ca4e42015-01-27 22:26:56 +0000184/// Computes instruction latency as max of latency of defined operands.
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000185///
186/// \param Root is a machine instruction that could be replaced by NewRoot.
187/// It is used to compute a more accurate latency information for NewRoot in
188/// case there is a dependent instruction in the same trace (\p BlockTrace)
189/// \param NewRoot is the instruction for which the latency is computed
190/// \param BlockTrace is a trace of machine instructions
191///
192/// \returns Latency of \p NewRoot
193unsigned MachineCombiner::getLatency(MachineInstr *Root, MachineInstr *NewRoot,
194 MachineTraceMetrics::Trace BlockTrace) {
Hal Finkele0fa8f22015-07-15 08:22:23 +0000195 assert(TSchedModel.hasInstrSchedModelOrItineraries() &&
196 "Missing machine model\n");
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000197
198 // Check each definition in NewRoot and compute the latency
199 unsigned NewRootLatency = 0;
200
Sanjay Patelf69f4e42015-05-21 17:43:26 +0000201 for (const MachineOperand &MO : NewRoot->operands()) {
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000202 // Check for virtual register operand.
203 if (!(MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg())))
204 continue;
205 if (!MO.isDef())
206 continue;
207 // Get the first instruction that uses MO
208 MachineRegisterInfo::reg_iterator RI = MRI->reg_begin(MO.getReg());
209 RI++;
210 MachineInstr *UseMO = RI->getParent();
211 unsigned LatencyOp = 0;
Duncan P. N. Exon Smithe59c8af2016-02-22 03:33:28 +0000212 if (UseMO && BlockTrace.isDepInTrace(*Root, *UseMO)) {
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000213 LatencyOp = TSchedModel.computeOperandLatency(
214 NewRoot, NewRoot->findRegisterDefOperandIdx(MO.getReg()), UseMO,
215 UseMO->findRegisterUseOperandIdx(MO.getReg()));
216 } else {
Hal Finkel17caf322015-08-05 07:45:28 +0000217 LatencyOp = TSchedModel.computeInstrLatency(NewRoot);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000218 }
219 NewRootLatency = std::max(NewRootLatency, LatencyOp);
220 }
221 return NewRootLatency;
222}
223
Sanjay Patel766589e2015-11-10 16:48:53 +0000224/// The combiner's goal may differ based on which pattern it is attempting
225/// to optimize.
226enum class CombinerObjective {
227 MustReduceDepth, // The data dependency chain must be improved.
228 Default // The critical path must not be lengthened.
229};
230
231static CombinerObjective getCombinerObjective(MachineCombinerPattern P) {
232 // TODO: If C++ ever gets a real enum class, make this part of the
233 // MachineCombinerPattern class.
234 switch (P) {
235 case MachineCombinerPattern::REASSOC_AX_BY:
236 case MachineCombinerPattern::REASSOC_AX_YB:
237 case MachineCombinerPattern::REASSOC_XA_BY:
238 case MachineCombinerPattern::REASSOC_XA_YB:
239 return CombinerObjective::MustReduceDepth;
240 default:
241 return CombinerObjective::Default;
242 }
243}
244
Sanjay Patele79b43a2015-06-23 00:39:40 +0000245/// The DAGCombine code sequence ends in MI (Machine Instruction) Root.
246/// The new code sequence ends in MI NewRoot. A necessary condition for the new
247/// sequence to replace the old sequence is that it cannot lengthen the critical
Sanjay Patel766589e2015-11-10 16:48:53 +0000248/// path. The definition of "improve" may be restricted by specifying that the
249/// new path improves the data dependency chain (MustReduceDepth).
Sanjay Patele79b43a2015-06-23 00:39:40 +0000250bool MachineCombiner::improvesCriticalPathLen(
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000251 MachineBasicBlock *MBB, MachineInstr *Root,
252 MachineTraceMetrics::Trace BlockTrace,
253 SmallVectorImpl<MachineInstr *> &InsInstrs,
Sebastian Pope08d9c72016-12-11 19:39:32 +0000254 SmallVectorImpl<MachineInstr *> &DelInstrs,
Sanjay Patele79b43a2015-06-23 00:39:40 +0000255 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
Florian Hahnceb44942017-09-20 11:54:37 +0000256 MachineCombinerPattern Pattern,
257 bool SlackIsAccurate) {
Hal Finkele0fa8f22015-07-15 08:22:23 +0000258 assert(TSchedModel.hasInstrSchedModelOrItineraries() &&
259 "Missing machine model\n");
Sanjay Patelccb8d5c2015-06-10 19:52:58 +0000260 // NewRoot is the last instruction in the \p InsInstrs vector.
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000261 unsigned NewRootIdx = InsInstrs.size() - 1;
262 MachineInstr *NewRoot = InsInstrs[NewRootIdx];
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000263
Sanjay Patel766589e2015-11-10 16:48:53 +0000264 // Get depth and latency of NewRoot and Root.
265 unsigned NewRootDepth = getDepth(InsInstrs, InstrIdxForVirtReg, BlockTrace);
Duncan P. N. Exon Smithe59c8af2016-02-22 03:33:28 +0000266 unsigned RootDepth = BlockTrace.getInstrCycles(*Root).Depth;
Sanjay Patel766589e2015-11-10 16:48:53 +0000267
Florian Hahnceb44942017-09-20 11:54:37 +0000268 DEBUG(dbgs() << "DEPENDENCE DATA FOR " << *Root << "\n";
Sanjay Patel766589e2015-11-10 16:48:53 +0000269 dbgs() << " NewRootDepth: " << NewRootDepth << "\n";
270 dbgs() << " RootDepth: " << RootDepth << "\n");
271
272 // For a transform such as reassociation, the cost equation is
273 // conservatively calculated so that we must improve the depth (data
274 // dependency cycles) in the critical path to proceed with the transform.
275 // Being conservative also protects against inaccuracies in the underlying
276 // machine trace metrics and CPU models.
277 if (getCombinerObjective(Pattern) == CombinerObjective::MustReduceDepth)
278 return NewRootDepth < RootDepth;
279
280 // A more flexible cost calculation for the critical path includes the slack
281 // of the original code sequence. This may allow the transform to proceed
282 // even if the instruction depths (data dependency cycles) become worse.
Sebastian Pope08d9c72016-12-11 19:39:32 +0000283
Florian Hahn001c3dd2017-12-06 20:27:33 +0000284 // Account for the latency of the inserted and deleted instructions by
285 // adding up their latencies. This assumes that the inserted and deleted
286 // instructions are dependent instruction chains, which might not hold
287 // in all cases.
288 unsigned NewRootLatency = 0;
289 for (unsigned i = 0; i < InsInstrs.size() - 1; i++)
290 NewRootLatency += TSchedModel.computeInstrLatency(InsInstrs[i]);
291 NewRootLatency += getLatency(Root, NewRoot, BlockTrace);
Sebastian Pope08d9c72016-12-11 19:39:32 +0000292
Florian Hahn001c3dd2017-12-06 20:27:33 +0000293 unsigned RootLatency = 0;
Sebastian Pope08d9c72016-12-11 19:39:32 +0000294 for (auto I : DelInstrs)
295 RootLatency += TSchedModel.computeInstrLatency(I);
296
Duncan P. N. Exon Smithe59c8af2016-02-22 03:33:28 +0000297 unsigned RootSlack = BlockTrace.getInstrSlack(*Root);
Florian Hahnceb44942017-09-20 11:54:37 +0000298 unsigned NewCycleCount = NewRootDepth + NewRootLatency;
299 unsigned OldCycleCount = RootDepth + RootLatency +
300 (SlackIsAccurate ? RootSlack : 0);
Sanjay Patel766589e2015-11-10 16:48:53 +0000301 DEBUG(dbgs() << " NewRootLatency: " << NewRootLatency << "\n";
302 dbgs() << " RootLatency: " << RootLatency << "\n";
Florian Hahnceb44942017-09-20 11:54:37 +0000303 dbgs() << " RootSlack: " << RootSlack << " SlackIsAccurate="
304 << SlackIsAccurate << "\n";
Sanjay Patelacd4bae2015-10-03 20:45:01 +0000305 dbgs() << " NewRootDepth + NewRootLatency = "
Florian Hahnceb44942017-09-20 11:54:37 +0000306 << NewCycleCount << "\n";
Sanjay Patelacd4bae2015-10-03 20:45:01 +0000307 dbgs() << " RootDepth + RootLatency + RootSlack = "
Florian Hahnceb44942017-09-20 11:54:37 +0000308 << OldCycleCount << "\n";
309 );
Junmo Park272a2bc2016-02-27 01:10:43 +0000310
Sanjay Patel766589e2015-11-10 16:48:53 +0000311 return NewCycleCount <= OldCycleCount;
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000312}
313
314/// helper routine to convert instructions into SC
315void MachineCombiner::instr2instrSC(
316 SmallVectorImpl<MachineInstr *> &Instrs,
317 SmallVectorImpl<const MCSchedClassDesc *> &InstrsSC) {
318 for (auto *InstrPtr : Instrs) {
319 unsigned Opc = InstrPtr->getOpcode();
320 unsigned Idx = TII->get(Opc).getSchedClass();
Pete Cooper11759452014-09-02 17:43:54 +0000321 const MCSchedClassDesc *SC = SchedModel.getSchedClassDesc(Idx);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000322 InstrsSC.push_back(SC);
323 }
324}
Hans Wennborg083ca9b2015-10-06 23:24:35 +0000325
Sanjay Patelb1ca4e42015-01-27 22:26:56 +0000326/// True when the new instructions do not increase resource length
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000327bool MachineCombiner::preservesResourceLen(
328 MachineBasicBlock *MBB, MachineTraceMetrics::Trace BlockTrace,
329 SmallVectorImpl<MachineInstr *> &InsInstrs,
330 SmallVectorImpl<MachineInstr *> &DelInstrs) {
Hal Finkele0fa8f22015-07-15 08:22:23 +0000331 if (!TSchedModel.hasInstrSchedModel())
332 return true;
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000333
334 // Compute current resource length
335
Gerolf Hoflehner97c383b2014-08-07 21:40:58 +0000336 //ArrayRef<const MachineBasicBlock *> MBBarr(MBB);
337 SmallVector <const MachineBasicBlock *, 1> MBBarr;
338 MBBarr.push_back(MBB);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000339 unsigned ResLenBeforeCombine = BlockTrace.getResourceLength(MBBarr);
340
341 // Deal with SC rather than Instructions.
342 SmallVector<const MCSchedClassDesc *, 16> InsInstrsSC;
343 SmallVector<const MCSchedClassDesc *, 16> DelInstrsSC;
344
345 instr2instrSC(InsInstrs, InsInstrsSC);
346 instr2instrSC(DelInstrs, DelInstrsSC);
347
348 ArrayRef<const MCSchedClassDesc *> MSCInsArr = makeArrayRef(InsInstrsSC);
349 ArrayRef<const MCSchedClassDesc *> MSCDelArr = makeArrayRef(DelInstrsSC);
350
Sanjay Patelccb8d5c2015-06-10 19:52:58 +0000351 // Compute new resource length.
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000352 unsigned ResLenAfterCombine =
353 BlockTrace.getResourceLength(MBBarr, MSCInsArr, MSCDelArr);
354
355 DEBUG(dbgs() << "RESOURCE DATA: \n";
356 dbgs() << " resource len before: " << ResLenBeforeCombine
357 << " after: " << ResLenAfterCombine << "\n";);
358
359 return ResLenAfterCombine <= ResLenBeforeCombine;
360}
361
362/// \returns true when new instruction sequence should be generated
Sanjay Patel6b280772015-01-27 22:16:52 +0000363/// independent if it lengthens critical path or not
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000364bool MachineCombiner::doSubstitute(unsigned NewSize, unsigned OldSize) {
365 if (OptSize && (NewSize < OldSize))
366 return true;
Hal Finkele0fa8f22015-07-15 08:22:23 +0000367 if (!TSchedModel.hasInstrSchedModelOrItineraries())
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000368 return true;
369 return false;
370}
371
Florian Hahnceb44942017-09-20 11:54:37 +0000372/// Inserts InsInstrs and deletes DelInstrs. Incrementally updates instruction
373/// depths if requested.
374///
375/// \param MBB basic block to insert instructions in
376/// \param MI current machine instruction
377/// \param InsInstrs new instructions to insert in \p MBB
378/// \param DelInstrs instruction to delete from \p MBB
379/// \param MinInstr is a pointer to the machine trace information
380/// \param RegUnits set of live registers, needed to compute instruction depths
381/// \param IncrementalUpdate if true, compute instruction depths incrementally,
382/// otherwise invalidate the trace
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000383static void insertDeleteInstructions(MachineBasicBlock *MBB, MachineInstr &MI,
384 SmallVector<MachineInstr *, 16> InsInstrs,
385 SmallVector<MachineInstr *, 16> DelInstrs,
Florian Hahnceb44942017-09-20 11:54:37 +0000386 MachineTraceMetrics::Ensemble *MinInstr,
387 SparseSet<LiveRegUnit> &RegUnits,
388 bool IncrementalUpdate) {
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000389 for (auto *InstrPtr : InsInstrs)
390 MBB->insert((MachineBasicBlock::iterator)&MI, InstrPtr);
Florian Hahnceb44942017-09-20 11:54:37 +0000391
392 for (auto *InstrPtr : DelInstrs) {
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000393 InstrPtr->eraseFromParentAndMarkDBGValuesForRemoval();
Florian Hahnceb44942017-09-20 11:54:37 +0000394 // Erase all LiveRegs defined by the removed instruction
395 for (auto I = RegUnits.begin(); I != RegUnits.end(); ) {
396 if (I->MI == InstrPtr)
397 I = RegUnits.erase(I);
398 else
399 I++;
400 }
401 }
402
403 if (IncrementalUpdate)
404 for (auto *InstrPtr : InsInstrs)
405 MinInstr->updateDepth(MBB, *InstrPtr, RegUnits);
406 else
407 MinInstr->invalidate(MBB);
408
409 NumInstCombined++;
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000410}
411
Sanjay Patelb1ca4e42015-01-27 22:26:56 +0000412/// Substitute a slow code sequence with a faster one by
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000413/// evaluating instruction combining pattern.
414/// The prototype of such a pattern is MUl + ADD -> MADD. Performs instruction
415/// combining based on machine trace metrics. Only combine a sequence of
416/// instructions when this neither lengthens the critical path nor increases
417/// resource pressure. When optimizing for codesize always combine when the new
418/// sequence is shorter.
419bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
420 bool Changed = false;
421 DEBUG(dbgs() << "Combining MBB " << MBB->getName() << "\n");
422
Florian Hahnceb44942017-09-20 11:54:37 +0000423 bool IncrementalUpdate = false;
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000424 auto BlockIter = MBB->begin();
Florian Hahne52abba2017-10-11 20:25:58 +0000425 decltype(BlockIter) LastUpdate;
Gerolf Hoflehner01b3a6182016-04-24 05:14:01 +0000426 // Check if the block is in a loop.
427 const MachineLoop *ML = MLI->getLoopFor(MBB);
Florian Hahnceb44942017-09-20 11:54:37 +0000428 if (!MinInstr)
429 MinInstr = Traces->getEnsemble(MachineTraceMetrics::TS_MinInstrCount);
430
431 SparseSet<LiveRegUnit> RegUnits;
432 RegUnits.setUniverse(TRI->getNumRegUnits());
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000433
434 while (BlockIter != MBB->end()) {
435 auto &MI = *BlockIter++;
436
437 DEBUG(dbgs() << "INSTR "; MI.dump(); dbgs() << "\n";);
Sanjay Patel387e66e2015-11-05 19:34:57 +0000438 SmallVector<MachineCombinerPattern, 16> Patterns;
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000439 // The motivating example is:
440 //
441 // MUL Other MUL_op1 MUL_op2 Other
442 // \ / \ | /
443 // ADD/SUB => MADD/MSUB
444 // (=Root) (=NewRoot)
445
446 // The DAGCombine code always replaced MUL + ADD/SUB by MADD. While this is
447 // usually beneficial for code size it unfortunately can hurt performance
448 // when the ADD is on the critical path, but the MUL is not. With the
449 // substitution the MUL becomes part of the critical path (in form of the
450 // MADD) and can lengthen it on architectures where the MADD latency is
451 // longer than the ADD latency.
452 //
453 // For each instruction we check if it can be the root of a combiner
454 // pattern. Then for each pattern the new code sequence in form of MI is
455 // generated and evaluated. When the efficiency criteria (don't lengthen
456 // critical path, don't use more resources) is met the new sequence gets
457 // hooked up into the basic block before the old sequence is removed.
458 //
459 // The algorithm does not try to evaluate all patterns and pick the best.
460 // This is only an artificial restriction though. In practice there is
Sanjay Patelcfe03932015-06-19 23:21:42 +0000461 // mostly one pattern, and getMachineCombinerPatterns() can order patterns
462 // based on an internal cost heuristic.
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000463
Sanjay Patel33ec5db2015-11-10 20:09:02 +0000464 if (!TII->getMachineCombinerPatterns(MI, Patterns))
465 continue;
466
467 for (auto P : Patterns) {
468 SmallVector<MachineInstr *, 16> InsInstrs;
469 SmallVector<MachineInstr *, 16> DelInstrs;
470 DenseMap<unsigned, unsigned> InstrIdxForVirtReg;
Sanjay Patel33ec5db2015-11-10 20:09:02 +0000471 TII->genAlternativeCodeSequence(MI, P, InsInstrs, DelInstrs,
472 InstrIdxForVirtReg);
473 unsigned NewInstCount = InsInstrs.size();
474 unsigned OldInstCount = DelInstrs.size();
475 // Found pattern, but did not generate alternative sequence.
476 // This can happen e.g. when an immediate could not be materialized
477 // in a single instruction.
478 if (!NewInstCount)
479 continue;
480
Gerolf Hoflehner01b3a6182016-04-24 05:14:01 +0000481 bool SubstituteAlways = false;
482 if (ML && TII->isThroughputPattern(P))
483 SubstituteAlways = true;
484
Florian Hahnceb44942017-09-20 11:54:37 +0000485 if (IncrementalUpdate) {
486 // Update depths since the last incremental update.
487 MinInstr->updateDepths(LastUpdate, BlockIter, RegUnits);
488 LastUpdate = BlockIter;
489 }
490
Sanjay Patel33ec5db2015-11-10 20:09:02 +0000491 // Substitute when we optimize for codesize and the new sequence has
492 // fewer instructions OR
493 // the new sequence neither lengthens the critical path nor increases
494 // resource pressure.
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000495 if (SubstituteAlways || doSubstitute(NewInstCount, OldInstCount)) {
Florian Hahnceb44942017-09-20 11:54:37 +0000496 insertDeleteInstructions(MBB, MI, InsInstrs, DelInstrs, MinInstr,
497 RegUnits, IncrementalUpdate);
Sanjay Patel33ec5db2015-11-10 20:09:02 +0000498 // Eagerly stop after the first pattern fires.
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000499 Changed = true;
Sanjay Patel33ec5db2015-11-10 20:09:02 +0000500 break;
501 } else {
Florian Hahnceb44942017-09-20 11:54:37 +0000502 // For big basic blocks, we only compute the full trace the first time
503 // we hit this. We do not invalidate the trace, but instead update the
504 // instruction depths incrementally.
505 // NOTE: Only the instruction depths up to MI are accurate. All other
506 // trace information is not updated.
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000507 MachineTraceMetrics::Trace BlockTrace = MinInstr->getTrace(MBB);
Florian Hahnceb44942017-09-20 11:54:37 +0000508 Traces->verifyAnalysis();
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000509 if (improvesCriticalPathLen(MBB, &MI, BlockTrace, InsInstrs, DelInstrs,
Florian Hahnceb44942017-09-20 11:54:37 +0000510 InstrIdxForVirtReg, P,
511 !IncrementalUpdate) &&
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000512 preservesResourceLen(MBB, BlockTrace, InsInstrs, DelInstrs)) {
Florian Hahne52abba2017-10-11 20:25:58 +0000513 if (MBB->size() > inc_threshold) {
Florian Hahnceb44942017-09-20 11:54:37 +0000514 // Use incremental depth updates for basic blocks above treshold
515 IncrementalUpdate = true;
Florian Hahne52abba2017-10-11 20:25:58 +0000516 LastUpdate = BlockIter;
517 }
Florian Hahnceb44942017-09-20 11:54:37 +0000518
519 insertDeleteInstructions(MBB, MI, InsInstrs, DelInstrs, MinInstr,
520 RegUnits, IncrementalUpdate);
521
Andrew V. Tischenko8da96912017-02-13 09:43:37 +0000522 // Eagerly stop after the first pattern fires.
523 Changed = true;
524 break;
525 }
Sanjay Patel33ec5db2015-11-10 20:09:02 +0000526 // Cleanup instructions of the alternative code sequence. There is no
527 // use for them.
528 MachineFunction *MF = MBB->getParent();
529 for (auto *InstrPtr : InsInstrs)
530 MF->DeleteMachineInstr(InstrPtr);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000531 }
Sanjay Patel33ec5db2015-11-10 20:09:02 +0000532 InstrIdxForVirtReg.clear();
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000533 }
534 }
535
Florian Hahnceb44942017-09-20 11:54:37 +0000536 if (Changed && IncrementalUpdate)
537 Traces->invalidate(MBB);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000538 return Changed;
539}
540
541bool MachineCombiner::runOnMachineFunction(MachineFunction &MF) {
Eric Christopher3d4276f2015-01-27 07:31:29 +0000542 const TargetSubtargetInfo &STI = MF.getSubtarget();
Eric Christopherd9134482014-08-04 21:25:23 +0000543 TII = STI.getInstrInfo();
544 TRI = STI.getRegisterInfo();
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000545 SchedModel = STI.getSchedModel();
Pete Cooper11759452014-09-02 17:43:54 +0000546 TSchedModel.init(SchedModel, &STI, TII);
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000547 MRI = &MF.getRegInfo();
Gerolf Hoflehner01b3a6182016-04-24 05:14:01 +0000548 MLI = &getAnalysis<MachineLoopInfo>();
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000549 Traces = &getAnalysis<MachineTraceMetrics>();
Hans Wennborg083ca9b2015-10-06 23:24:35 +0000550 MinInstr = nullptr;
Matthias Braunf1caa282017-12-15 22:22:58 +0000551 OptSize = MF.getFunction().optForSize();
Gerolf Hoflehner5e1207e2014-08-03 21:35:39 +0000552
553 DEBUG(dbgs() << getPassName() << ": " << MF.getName() << '\n');
554 if (!TII->useMachineCombiner()) {
555 DEBUG(dbgs() << " Skipping pass: Target does not support machine combiner\n");
556 return false;
557 }
558
559 bool Changed = false;
560
561 // Try to combine instructions.
562 for (auto &MBB : MF)
563 Changed |= combineInstructions(&MBB);
564
565 return Changed;
566}