blob: ae26967b235e8fab483674c6a0d7604ea5fea42e [file] [log] [blame]
Evan Chengc6fe3332010-03-02 02:38:24 +00001//===-- MachineCSE.cpp - Machine Common Subexpression Elimination Pass ----===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass performs global common subexpression elimination on machine
Evan Chengc5bbba12010-03-02 19:02:27 +000011// instructions using a scoped hash table based value numbering scheme. It
Evan Chengc6fe3332010-03-02 02:38:24 +000012// must be run while the machine function is still in SSA form.
13//
14//===----------------------------------------------------------------------===//
15
Evan Chengc6fe3332010-03-02 02:38:24 +000016#include "llvm/CodeGen/Passes.h"
Evan Cheng31156982010-04-21 00:21:07 +000017#include "llvm/ADT/DenseMap.h"
Evan Chengc6fe3332010-03-02 02:38:24 +000018#include "llvm/ADT/ScopedHashTable.h"
Evan Cheng189c1ec2010-10-29 23:36:03 +000019#include "llvm/ADT/SmallSet.h"
Evan Chengc6fe3332010-03-02 02:38:24 +000020#include "llvm/ADT/Statistic.h"
Chandler Carruthd04a8d42012-12-03 16:50:05 +000021#include "llvm/Analysis/AliasAnalysis.h"
22#include "llvm/CodeGen/MachineDominators.h"
23#include "llvm/CodeGen/MachineInstr.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
Evan Chengc6fe3332010-03-02 02:38:24 +000025#include "llvm/Support/Debug.h"
Cameron Zwarich53eeba52011-01-03 04:07:46 +000026#include "llvm/Support/RecyclingAllocator.h"
Chandler Carruthd04a8d42012-12-03 16:50:05 +000027#include "llvm/Target/TargetInstrInfo.h"
Stephen Hines37ed9c12014-12-01 14:51:49 -080028#include "llvm/Target/TargetSubtargetInfo.h"
Evan Chengc6fe3332010-03-02 02:38:24 +000029using namespace llvm;
30
Stephen Hinesdce4a402014-05-29 02:49:00 -070031#define DEBUG_TYPE "machine-cse"
32
Evan Cheng16b48b82010-03-03 21:20:05 +000033STATISTIC(NumCoalesces, "Number of copies coalesced");
34STATISTIC(NumCSEs, "Number of common subexpression eliminated");
Evan Cheng189c1ec2010-10-29 23:36:03 +000035STATISTIC(NumPhysCSEs,
36 "Number of physreg referencing common subexpr eliminated");
Evan Cheng97b5beb2012-01-10 02:02:58 +000037STATISTIC(NumCrossBBCSEs,
38 "Number of cross-MBB physreg referencing CS eliminated");
Evan Chenga63cde22010-12-15 22:16:21 +000039STATISTIC(NumCommutes, "Number of copies coalesced after commuting");
Bob Wilson38441732010-06-03 18:28:31 +000040
Evan Chengc6fe3332010-03-02 02:38:24 +000041namespace {
42 class MachineCSE : public MachineFunctionPass {
Evan Cheng6ba95542010-03-03 02:48:20 +000043 const TargetInstrInfo *TII;
Evan Chengb3958e82010-03-04 01:33:55 +000044 const TargetRegisterInfo *TRI;
Evan Chenga5f32cb2010-03-04 21:18:08 +000045 AliasAnalysis *AA;
Evan Cheng31f94c72010-03-09 03:21:12 +000046 MachineDominatorTree *DT;
47 MachineRegisterInfo *MRI;
Evan Chengc6fe3332010-03-02 02:38:24 +000048 public:
49 static char ID; // Pass identification
Owen Anderson081c34b2010-10-19 17:21:58 +000050 MachineCSE() : MachineFunctionPass(ID), LookAheadLimit(5), CurrVN(0) {
51 initializeMachineCSEPass(*PassRegistry::getPassRegistry());
52 }
Evan Chengc6fe3332010-03-02 02:38:24 +000053
Stephen Hines36b56882014-04-23 16:57:46 -070054 bool runOnMachineFunction(MachineFunction &MF) override;
Andrew Trick1df91b02012-02-08 21:22:43 +000055
Stephen Hines36b56882014-04-23 16:57:46 -070056 void getAnalysisUsage(AnalysisUsage &AU) const override {
Evan Chengc6fe3332010-03-02 02:38:24 +000057 AU.setPreservesCFG();
58 MachineFunctionPass::getAnalysisUsage(AU);
Evan Chenga5f32cb2010-03-04 21:18:08 +000059 AU.addRequired<AliasAnalysis>();
Evan Cheng65424162010-08-17 20:57:42 +000060 AU.addPreservedID(MachineLoopInfoID);
Evan Chengc6fe3332010-03-02 02:38:24 +000061 AU.addRequired<MachineDominatorTree>();
62 AU.addPreserved<MachineDominatorTree>();
63 }
64
Stephen Hines36b56882014-04-23 16:57:46 -070065 void releaseMemory() override {
Evan Chengc2b768f2010-09-17 21:59:42 +000066 ScopeMap.clear();
67 Exps.clear();
68 }
69
Evan Chengc6fe3332010-03-02 02:38:24 +000070 private:
Evan Cheng835810b2010-05-21 21:22:19 +000071 const unsigned LookAheadLimit;
Cameron Zwarich53eeba52011-01-03 04:07:46 +000072 typedef RecyclingAllocator<BumpPtrAllocator,
73 ScopedHashTableVal<MachineInstr*, unsigned> > AllocatorTy;
74 typedef ScopedHashTable<MachineInstr*, unsigned,
75 MachineInstrExpressionTrait, AllocatorTy> ScopedHTType;
76 typedef ScopedHTType::ScopeTy ScopeType;
Evan Cheng31156982010-04-21 00:21:07 +000077 DenseMap<MachineBasicBlock*, ScopeType*> ScopeMap;
Cameron Zwarich53eeba52011-01-03 04:07:46 +000078 ScopedHTType VNT;
Evan Cheng16b48b82010-03-03 21:20:05 +000079 SmallVector<MachineInstr*, 64> Exps;
Evan Cheng31156982010-04-21 00:21:07 +000080 unsigned CurrVN;
Evan Cheng16b48b82010-03-03 21:20:05 +000081
Stephen Hines37ed9c12014-12-01 14:51:49 -080082 bool PerformTrivialCopyPropagation(MachineInstr *MI,
83 MachineBasicBlock *MBB);
Evan Chengb3958e82010-03-04 01:33:55 +000084 bool isPhysDefTriviallyDead(unsigned Reg,
85 MachineBasicBlock::const_iterator I,
Nick Lewycky7a7a6db2012-07-05 06:19:21 +000086 MachineBasicBlock::const_iterator E) const;
Evan Cheng189c1ec2010-10-29 23:36:03 +000087 bool hasLivePhysRegDefUses(const MachineInstr *MI,
88 const MachineBasicBlock *MBB,
Evan Cheng97b5beb2012-01-10 02:02:58 +000089 SmallSet<unsigned,8> &PhysRefs,
Craig Toppera0ec3f92013-07-14 04:42:23 +000090 SmallVectorImpl<unsigned> &PhysDefs,
Ulrich Weigandb64e2112012-11-13 18:40:58 +000091 bool &PhysUseDef) const;
Evan Cheng189c1ec2010-10-29 23:36:03 +000092 bool PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI,
Evan Cheng97b5beb2012-01-10 02:02:58 +000093 SmallSet<unsigned,8> &PhysRefs,
Craig Toppera0ec3f92013-07-14 04:42:23 +000094 SmallVectorImpl<unsigned> &PhysDefs,
Evan Cheng97b5beb2012-01-10 02:02:58 +000095 bool &NonLocal) const;
Evan Chenga5f32cb2010-03-04 21:18:08 +000096 bool isCSECandidate(MachineInstr *MI);
Evan Cheng2938a002010-03-10 02:12:03 +000097 bool isProfitableToCSE(unsigned CSReg, unsigned Reg,
98 MachineInstr *CSMI, MachineInstr *MI);
Evan Cheng31156982010-04-21 00:21:07 +000099 void EnterScope(MachineBasicBlock *MBB);
100 void ExitScope(MachineBasicBlock *MBB);
101 bool ProcessBlock(MachineBasicBlock *MBB);
102 void ExitScopeIfDone(MachineDomTreeNode *Node,
Bill Wendling96cb1122012-07-19 00:04:14 +0000103 DenseMap<MachineDomTreeNode*, unsigned> &OpenChildren);
Evan Cheng31156982010-04-21 00:21:07 +0000104 bool PerformCSE(MachineDomTreeNode *Node);
Evan Chengc6fe3332010-03-02 02:38:24 +0000105 };
106} // end anonymous namespace
107
108char MachineCSE::ID = 0;
Andrew Trick1dd8c852012-02-08 21:23:13 +0000109char &llvm::MachineCSEID = MachineCSE::ID;
Owen Anderson2ab36d32010-10-12 19:48:12 +0000110INITIALIZE_PASS_BEGIN(MachineCSE, "machine-cse",
111 "Machine Common Subexpression Elimination", false, false)
112INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
113INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
114INITIALIZE_PASS_END(MachineCSE, "machine-cse",
Owen Andersonce665bd2010-10-07 22:25:06 +0000115 "Machine Common Subexpression Elimination", false, false)
Evan Chengc6fe3332010-03-02 02:38:24 +0000116
Stephen Hines37ed9c12014-12-01 14:51:49 -0800117/// The source register of a COPY machine instruction can be propagated to all
118/// its users, and this propagation could increase the probability of finding
119/// common subexpressions. If the COPY has only one user, the COPY itself can
120/// be removed.
121bool MachineCSE::PerformTrivialCopyPropagation(MachineInstr *MI,
122 MachineBasicBlock *MBB) {
Evan Cheng6ba95542010-03-03 02:48:20 +0000123 bool Changed = false;
124 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
125 MachineOperand &MO = MI->getOperand(i);
Evan Cheng16b48b82010-03-03 21:20:05 +0000126 if (!MO.isReg() || !MO.isUse())
127 continue;
128 unsigned Reg = MO.getReg();
Jakob Stoklund Olesenc9df0252011-01-10 02:58:51 +0000129 if (!TargetRegisterInfo::isVirtualRegister(Reg))
Evan Cheng16b48b82010-03-03 21:20:05 +0000130 continue;
Stephen Hines37ed9c12014-12-01 14:51:49 -0800131 bool OnlyOneUse = MRI->hasOneNonDBGUse(Reg);
Evan Cheng16b48b82010-03-03 21:20:05 +0000132 MachineInstr *DefMI = MRI->getVRegDef(Reg);
Jakob Stoklund Olesen0bc25f42010-07-08 16:40:22 +0000133 if (!DefMI->isCopy())
134 continue;
Jakob Stoklund Olesen04c528a2010-07-16 04:45:42 +0000135 unsigned SrcReg = DefMI->getOperand(1).getReg();
Jakob Stoklund Olesen0bc25f42010-07-08 16:40:22 +0000136 if (!TargetRegisterInfo::isVirtualRegister(SrcReg))
137 continue;
Stephen Hines36b56882014-04-23 16:57:46 -0700138 if (DefMI->getOperand(0).getSubReg())
Jakob Stoklund Olesen0bc25f42010-07-08 16:40:22 +0000139 continue;
Stephen Hines36b56882014-04-23 16:57:46 -0700140 // FIXME: We should trivially coalesce subregister copies to expose CSE
141 // opportunities on instructions with truncated operands (see
142 // cse-add-with-overflow.ll). This can be done here as follows:
143 // if (SrcSubReg)
144 // RC = TRI->getMatchingSuperRegClass(MRI->getRegClass(SrcReg), RC,
145 // SrcSubReg);
146 // MO.substVirtReg(SrcReg, SrcSubReg, *TRI);
147 //
148 // The 2-addr pass has been updated to handle coalesced subregs. However,
149 // some machine-specific code still can't handle it.
150 // To handle it properly we also need a way find a constrained subregister
151 // class given a super-reg class and subreg index.
152 if (DefMI->getOperand(1).getSubReg())
153 continue;
154 const TargetRegisterClass *RC = MRI->getRegClass(Reg);
155 if (!MRI->constrainRegClass(SrcReg, RC))
Jakob Stoklund Olesen0bc25f42010-07-08 16:40:22 +0000156 continue;
157 DEBUG(dbgs() << "Coalescing: " << *DefMI);
Jakob Stoklund Olesenbf4699c2010-10-06 23:54:39 +0000158 DEBUG(dbgs() << "*** to: " << *MI);
Stephen Hines37ed9c12014-12-01 14:51:49 -0800159 // Propagate SrcReg of copies to MI.
Jakob Stoklund Olesen0bc25f42010-07-08 16:40:22 +0000160 MO.setReg(SrcReg);
161 MRI->clearKillFlags(SrcReg);
Stephen Hines37ed9c12014-12-01 14:51:49 -0800162 // Coalesce single use copies.
163 if (OnlyOneUse) {
164 DefMI->eraseFromParent();
165 ++NumCoalesces;
166 }
Jakob Stoklund Olesen0bc25f42010-07-08 16:40:22 +0000167 Changed = true;
Evan Cheng6ba95542010-03-03 02:48:20 +0000168 }
169
170 return Changed;
171}
172
Evan Cheng835810b2010-05-21 21:22:19 +0000173bool
174MachineCSE::isPhysDefTriviallyDead(unsigned Reg,
175 MachineBasicBlock::const_iterator I,
176 MachineBasicBlock::const_iterator E) const {
Eric Christophere81d0102010-05-21 23:40:03 +0000177 unsigned LookAheadLeft = LookAheadLimit;
Evan Cheng112e5e72010-03-23 20:33:48 +0000178 while (LookAheadLeft) {
Evan Cheng22504252010-03-24 01:50:28 +0000179 // Skip over dbg_value's.
180 while (I != E && I->isDebugValue())
181 ++I;
182
Evan Chengb3958e82010-03-04 01:33:55 +0000183 if (I == E)
184 // Reached end of block, register is obviously dead.
185 return true;
186
Evan Chengb3958e82010-03-04 01:33:55 +0000187 bool SeenDef = false;
188 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
189 const MachineOperand &MO = I->getOperand(i);
Jakob Stoklund Olesen2129a0f2012-02-28 02:08:50 +0000190 if (MO.isRegMask() && MO.clobbersPhysReg(Reg))
191 SeenDef = true;
Evan Chengb3958e82010-03-04 01:33:55 +0000192 if (!MO.isReg() || !MO.getReg())
193 continue;
194 if (!TRI->regsOverlap(MO.getReg(), Reg))
195 continue;
196 if (MO.isUse())
Evan Cheng835810b2010-05-21 21:22:19 +0000197 // Found a use!
Evan Chengb3958e82010-03-04 01:33:55 +0000198 return false;
199 SeenDef = true;
200 }
201 if (SeenDef)
Andrew Trick1df91b02012-02-08 21:22:43 +0000202 // See a def of Reg (or an alias) before encountering any use, it's
Evan Chengb3958e82010-03-04 01:33:55 +0000203 // trivially dead.
204 return true;
Evan Cheng112e5e72010-03-23 20:33:48 +0000205
206 --LookAheadLeft;
Evan Chengb3958e82010-03-04 01:33:55 +0000207 ++I;
208 }
209 return false;
210}
211
Evan Cheng189c1ec2010-10-29 23:36:03 +0000212/// hasLivePhysRegDefUses - Return true if the specified instruction read/write
Evan Cheng835810b2010-05-21 21:22:19 +0000213/// physical registers (except for dead defs of physical registers). It also
Evan Cheng2b4e7272010-06-04 23:28:13 +0000214/// returns the physical register def by reference if it's the only one and the
215/// instruction does not uses a physical register.
Evan Cheng189c1ec2010-10-29 23:36:03 +0000216bool MachineCSE::hasLivePhysRegDefUses(const MachineInstr *MI,
217 const MachineBasicBlock *MBB,
Evan Cheng97b5beb2012-01-10 02:02:58 +0000218 SmallSet<unsigned,8> &PhysRefs,
Craig Toppera0ec3f92013-07-14 04:42:23 +0000219 SmallVectorImpl<unsigned> &PhysDefs,
Ulrich Weigandb64e2112012-11-13 18:40:58 +0000220 bool &PhysUseDef) const{
221 // First, add all uses to PhysRefs.
Evan Cheng6ba95542010-03-03 02:48:20 +0000222 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
Evan Cheng835810b2010-05-21 21:22:19 +0000223 const MachineOperand &MO = MI->getOperand(i);
Ulrich Weigandb64e2112012-11-13 18:40:58 +0000224 if (!MO.isReg() || MO.isDef())
Evan Cheng6ba95542010-03-03 02:48:20 +0000225 continue;
226 unsigned Reg = MO.getReg();
227 if (!Reg)
228 continue;
Evan Cheng835810b2010-05-21 21:22:19 +0000229 if (TargetRegisterInfo::isVirtualRegister(Reg))
230 continue;
Benjamin Kramer5fa2d452012-08-11 20:42:59 +0000231 // Reading constant physregs is ok.
232 if (!MRI->isConstantPhysReg(Reg, *MBB->getParent()))
233 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
Benjamin Kramercfc0ad62012-08-11 19:05:13 +0000234 PhysRefs.insert(*AI);
Ulrich Weigandb64e2112012-11-13 18:40:58 +0000235 }
236
237 // Next, collect all defs into PhysDefs. If any is already in PhysRefs
238 // (which currently contains only uses), set the PhysUseDef flag.
239 PhysUseDef = false;
Stephen Hines36b56882014-04-23 16:57:46 -0700240 MachineBasicBlock::const_iterator I = MI; I = std::next(I);
Ulrich Weigandb64e2112012-11-13 18:40:58 +0000241 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
242 const MachineOperand &MO = MI->getOperand(i);
243 if (!MO.isReg() || !MO.isDef())
244 continue;
245 unsigned Reg = MO.getReg();
246 if (!Reg)
247 continue;
248 if (TargetRegisterInfo::isVirtualRegister(Reg))
249 continue;
250 // Check against PhysRefs even if the def is "dead".
251 if (PhysRefs.count(Reg))
252 PhysUseDef = true;
253 // If the def is dead, it's ok. But the def may not marked "dead". That's
254 // common since this pass is run before livevariables. We can scan
255 // forward a few instructions and check if it is obviously dead.
256 if (!MO.isDead() && !isPhysDefTriviallyDead(Reg, I, MBB->end()))
Evan Cheng97b5beb2012-01-10 02:02:58 +0000257 PhysDefs.push_back(Reg);
Evan Chengb3958e82010-03-04 01:33:55 +0000258 }
259
Ulrich Weigandb64e2112012-11-13 18:40:58 +0000260 // Finally, add all defs to PhysRefs as well.
261 for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i)
262 for (MCRegAliasIterator AI(PhysDefs[i], TRI, true); AI.isValid(); ++AI)
263 PhysRefs.insert(*AI);
264
Evan Cheng189c1ec2010-10-29 23:36:03 +0000265 return !PhysRefs.empty();
Evan Chengc6fe3332010-03-02 02:38:24 +0000266}
267
Evan Cheng189c1ec2010-10-29 23:36:03 +0000268bool MachineCSE::PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI,
Evan Cheng97b5beb2012-01-10 02:02:58 +0000269 SmallSet<unsigned,8> &PhysRefs,
Craig Toppera0ec3f92013-07-14 04:42:23 +0000270 SmallVectorImpl<unsigned> &PhysDefs,
Evan Cheng97b5beb2012-01-10 02:02:58 +0000271 bool &NonLocal) const {
Eli Friedman5e926ac2011-05-06 05:23:07 +0000272 // For now conservatively returns false if the common subexpression is
Evan Cheng97b5beb2012-01-10 02:02:58 +0000273 // not in the same basic block as the given instruction. The only exception
274 // is if the common subexpression is in the sole predecessor block.
275 const MachineBasicBlock *MBB = MI->getParent();
276 const MachineBasicBlock *CSMBB = CSMI->getParent();
277
278 bool CrossMBB = false;
279 if (CSMBB != MBB) {
Evan Chengf96703e2012-01-11 00:38:11 +0000280 if (MBB->pred_size() != 1 || *MBB->pred_begin() != CSMBB)
Evan Cheng97b5beb2012-01-10 02:02:58 +0000281 return false;
Evan Chengf96703e2012-01-11 00:38:11 +0000282
283 for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i) {
Jakob Stoklund Olesenfb9ebbf2012-10-15 21:57:41 +0000284 if (MRI->isAllocatable(PhysDefs[i]) || MRI->isReserved(PhysDefs[i]))
Lang Hamesc2e08db2012-02-17 00:27:16 +0000285 // Avoid extending live range of physical registers if they are
286 //allocatable or reserved.
Evan Chengf96703e2012-01-11 00:38:11 +0000287 return false;
288 }
289 CrossMBB = true;
Evan Cheng97b5beb2012-01-10 02:02:58 +0000290 }
Stephen Hines36b56882014-04-23 16:57:46 -0700291 MachineBasicBlock::const_iterator I = CSMI; I = std::next(I);
Eli Friedman5e926ac2011-05-06 05:23:07 +0000292 MachineBasicBlock::const_iterator E = MI;
Evan Cheng97b5beb2012-01-10 02:02:58 +0000293 MachineBasicBlock::const_iterator EE = CSMBB->end();
Evan Cheng835810b2010-05-21 21:22:19 +0000294 unsigned LookAheadLeft = LookAheadLimit;
295 while (LookAheadLeft) {
Eli Friedman5e926ac2011-05-06 05:23:07 +0000296 // Skip over dbg_value's.
Evan Cheng97b5beb2012-01-10 02:02:58 +0000297 while (I != E && I != EE && I->isDebugValue())
Evan Cheng835810b2010-05-21 21:22:19 +0000298 ++I;
Eli Friedman5e926ac2011-05-06 05:23:07 +0000299
Evan Cheng97b5beb2012-01-10 02:02:58 +0000300 if (I == EE) {
301 assert(CrossMBB && "Reaching end-of-MBB without finding MI?");
Duncan Sands5b8a1db2012-02-05 14:20:11 +0000302 (void)CrossMBB;
Evan Cheng97b5beb2012-01-10 02:02:58 +0000303 CrossMBB = false;
304 NonLocal = true;
305 I = MBB->begin();
306 EE = MBB->end();
307 continue;
308 }
309
Eli Friedman5e926ac2011-05-06 05:23:07 +0000310 if (I == E)
311 return true;
312
313 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
314 const MachineOperand &MO = I->getOperand(i);
Jakob Stoklund Olesen2129a0f2012-02-28 02:08:50 +0000315 // RegMasks go on instructions like calls that clobber lots of physregs.
316 // Don't attempt to CSE across such an instruction.
317 if (MO.isRegMask())
318 return false;
Eli Friedman5e926ac2011-05-06 05:23:07 +0000319 if (!MO.isReg() || !MO.isDef())
320 continue;
321 unsigned MOReg = MO.getReg();
322 if (TargetRegisterInfo::isVirtualRegister(MOReg))
323 continue;
324 if (PhysRefs.count(MOReg))
325 return false;
Evan Cheng189c1ec2010-10-29 23:36:03 +0000326 }
Eli Friedman5e926ac2011-05-06 05:23:07 +0000327
328 --LookAheadLeft;
329 ++I;
Evan Cheng835810b2010-05-21 21:22:19 +0000330 }
331
332 return false;
333}
334
Evan Chenga5f32cb2010-03-04 21:18:08 +0000335bool MachineCSE::isCSECandidate(MachineInstr *MI) {
Stephen Hines36b56882014-04-23 16:57:46 -0700336 if (MI->isPosition() || MI->isPHI() || MI->isImplicitDef() || MI->isKill() ||
337 MI->isInlineAsm() || MI->isDebugValue())
Evan Cheng51960182010-03-08 23:49:12 +0000338 return false;
339
Evan Cheng2938a002010-03-10 02:12:03 +0000340 // Ignore copies.
Jakob Stoklund Olesen04c528a2010-07-16 04:45:42 +0000341 if (MI->isCopyLike())
Evan Chenga5f32cb2010-03-04 21:18:08 +0000342 return false;
343
344 // Ignore stuff that we obviously can't move.
Evan Cheng5a96b3d2011-12-07 07:15:52 +0000345 if (MI->mayStore() || MI->isCall() || MI->isTerminator() ||
Evan Chengc36b7062011-01-07 23:50:32 +0000346 MI->hasUnmodeledSideEffects())
Evan Chenga5f32cb2010-03-04 21:18:08 +0000347 return false;
348
Evan Cheng5a96b3d2011-12-07 07:15:52 +0000349 if (MI->mayLoad()) {
Evan Chenga5f32cb2010-03-04 21:18:08 +0000350 // Okay, this instruction does a load. As a refinement, we allow the target
351 // to decide whether the loaded value is actually a constant. If so, we can
352 // actually use it as a load.
353 if (!MI->isInvariantLoad(AA))
354 // FIXME: we should be able to hoist loads with no other side effects if
355 // there are no other instructions which can change memory in this loop.
356 // This is a trivial form of alias analysis.
357 return false;
358 }
359 return true;
360}
361
Evan Cheng31f94c72010-03-09 03:21:12 +0000362/// isProfitableToCSE - Return true if it's profitable to eliminate MI with a
363/// common expression that defines Reg.
Evan Cheng2938a002010-03-10 02:12:03 +0000364bool MachineCSE::isProfitableToCSE(unsigned CSReg, unsigned Reg,
365 MachineInstr *CSMI, MachineInstr *MI) {
366 // FIXME: Heuristics that works around the lack the live range splitting.
367
Manman Renba86b132012-08-07 06:16:46 +0000368 // If CSReg is used at all uses of Reg, CSE should not increase register
369 // pressure of CSReg.
370 bool MayIncreasePressure = true;
371 if (TargetRegisterInfo::isVirtualRegister(CSReg) &&
372 TargetRegisterInfo::isVirtualRegister(Reg)) {
373 MayIncreasePressure = false;
374 SmallPtrSet<MachineInstr*, 8> CSUses;
Stephen Hines36b56882014-04-23 16:57:46 -0700375 for (MachineInstr &MI : MRI->use_nodbg_instructions(CSReg)) {
376 CSUses.insert(&MI);
Manman Renba86b132012-08-07 06:16:46 +0000377 }
Stephen Hines36b56882014-04-23 16:57:46 -0700378 for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) {
379 if (!CSUses.count(&MI)) {
Manman Renba86b132012-08-07 06:16:46 +0000380 MayIncreasePressure = true;
381 break;
382 }
383 }
384 }
385 if (!MayIncreasePressure) return true;
386
Chris Lattner622a11b2011-01-10 07:51:31 +0000387 // Heuristics #1: Don't CSE "cheap" computation if the def is not local or in
388 // an immediate predecessor. We don't want to increase register pressure and
389 // end up causing other computation to be spilled.
Stephen Hinesbfc2d682014-10-17 08:47:43 -0700390 if (TII->isAsCheapAsAMove(MI)) {
Evan Cheng2938a002010-03-10 02:12:03 +0000391 MachineBasicBlock *CSBB = CSMI->getParent();
392 MachineBasicBlock *BB = MI->getParent();
Chris Lattner622a11b2011-01-10 07:51:31 +0000393 if (CSBB != BB && !CSBB->isSuccessor(BB))
Evan Cheng2938a002010-03-10 02:12:03 +0000394 return false;
395 }
396
397 // Heuristics #2: If the expression doesn't not use a vr and the only use
398 // of the redundant computation are copies, do not cse.
399 bool HasVRegUse = false;
400 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
401 const MachineOperand &MO = MI->getOperand(i);
Jakob Stoklund Olesenc9df0252011-01-10 02:58:51 +0000402 if (MO.isReg() && MO.isUse() &&
Evan Cheng2938a002010-03-10 02:12:03 +0000403 TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
404 HasVRegUse = true;
405 break;
406 }
407 }
408 if (!HasVRegUse) {
409 bool HasNonCopyUse = false;
Stephen Hines36b56882014-04-23 16:57:46 -0700410 for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) {
Evan Cheng2938a002010-03-10 02:12:03 +0000411 // Ignore copies.
Stephen Hines36b56882014-04-23 16:57:46 -0700412 if (!MI.isCopyLike()) {
Evan Cheng2938a002010-03-10 02:12:03 +0000413 HasNonCopyUse = true;
414 break;
415 }
416 }
417 if (!HasNonCopyUse)
418 return false;
419 }
420
421 // Heuristics #3: If the common subexpression is used by PHIs, do not reuse
422 // it unless the defined value is already used in the BB of the new use.
Evan Cheng31f94c72010-03-09 03:21:12 +0000423 bool HasPHI = false;
424 SmallPtrSet<MachineBasicBlock*, 4> CSBBs;
Stephen Hines36b56882014-04-23 16:57:46 -0700425 for (MachineInstr &MI : MRI->use_nodbg_instructions(CSReg)) {
426 HasPHI |= MI.isPHI();
427 CSBBs.insert(MI.getParent());
Evan Cheng31f94c72010-03-09 03:21:12 +0000428 }
429
430 if (!HasPHI)
431 return true;
432 return CSBBs.count(MI->getParent());
433}
434
Evan Cheng31156982010-04-21 00:21:07 +0000435void MachineCSE::EnterScope(MachineBasicBlock *MBB) {
436 DEBUG(dbgs() << "Entering: " << MBB->getName() << '\n');
437 ScopeType *Scope = new ScopeType(VNT);
438 ScopeMap[MBB] = Scope;
439}
440
441void MachineCSE::ExitScope(MachineBasicBlock *MBB) {
442 DEBUG(dbgs() << "Exiting: " << MBB->getName() << '\n');
443 DenseMap<MachineBasicBlock*, ScopeType*>::iterator SI = ScopeMap.find(MBB);
444 assert(SI != ScopeMap.end());
Evan Cheng31156982010-04-21 00:21:07 +0000445 delete SI->second;
Jakub Staszakbb8ddc72012-11-26 22:14:19 +0000446 ScopeMap.erase(SI);
Evan Cheng31156982010-04-21 00:21:07 +0000447}
448
449bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) {
Evan Cheng6ba95542010-03-03 02:48:20 +0000450 bool Changed = false;
451
Evan Cheng31f94c72010-03-09 03:21:12 +0000452 SmallVector<std::pair<unsigned, unsigned>, 8> CSEPairs;
Manman Ren39ad5682012-08-08 00:51:41 +0000453 SmallVector<unsigned, 2> ImplicitDefsToUpdate;
Evan Cheng16b48b82010-03-03 21:20:05 +0000454 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E; ) {
Evan Cheng6ba95542010-03-03 02:48:20 +0000455 MachineInstr *MI = &*I;
Evan Cheng16b48b82010-03-03 21:20:05 +0000456 ++I;
Evan Chenga5f32cb2010-03-04 21:18:08 +0000457
458 if (!isCSECandidate(MI))
Evan Cheng6ba95542010-03-03 02:48:20 +0000459 continue;
Evan Cheng6ba95542010-03-03 02:48:20 +0000460
461 bool FoundCSE = VNT.count(MI);
462 if (!FoundCSE) {
Stephen Hines37ed9c12014-12-01 14:51:49 -0800463 // Using trivial copy propagation to find more CSE opportunities.
464 if (PerformTrivialCopyPropagation(MI, MBB)) {
Evan Chengcfea9852011-04-11 18:47:20 +0000465 Changed = true;
466
Evan Chengdb8771a2010-04-02 02:21:24 +0000467 // After coalescing MI itself may become a copy.
Jakob Stoklund Olesen04c528a2010-07-16 04:45:42 +0000468 if (MI->isCopyLike())
Evan Chengdb8771a2010-04-02 02:21:24 +0000469 continue;
Stephen Hines37ed9c12014-12-01 14:51:49 -0800470
471 // Try again to see if CSE is possible.
Evan Cheng6ba95542010-03-03 02:48:20 +0000472 FoundCSE = VNT.count(MI);
Evan Chengdb8771a2010-04-02 02:21:24 +0000473 }
Evan Cheng6ba95542010-03-03 02:48:20 +0000474 }
Evan Chenga63cde22010-12-15 22:16:21 +0000475
476 // Commute commutable instructions.
477 bool Commuted = false;
Evan Cheng5a96b3d2011-12-07 07:15:52 +0000478 if (!FoundCSE && MI->isCommutable()) {
Evan Chenga63cde22010-12-15 22:16:21 +0000479 MachineInstr *NewMI = TII->commuteInstruction(MI);
480 if (NewMI) {
481 Commuted = true;
482 FoundCSE = VNT.count(NewMI);
Evan Chengcfea9852011-04-11 18:47:20 +0000483 if (NewMI != MI) {
Evan Chenga63cde22010-12-15 22:16:21 +0000484 // New instruction. It doesn't need to be kept.
485 NewMI->eraseFromParent();
Evan Chengcfea9852011-04-11 18:47:20 +0000486 Changed = true;
487 } else if (!FoundCSE)
Evan Chenga63cde22010-12-15 22:16:21 +0000488 // MI was changed but it didn't help, commute it back!
489 (void)TII->commuteInstruction(MI);
490 }
491 }
Evan Cheng6ba95542010-03-03 02:48:20 +0000492
Evan Cheng189c1ec2010-10-29 23:36:03 +0000493 // If the instruction defines physical registers and the values *may* be
Evan Cheng67bda722010-03-03 23:59:08 +0000494 // used, then it's not safe to replace it with a common subexpression.
Evan Cheng189c1ec2010-10-29 23:36:03 +0000495 // It's also not safe if the instruction uses physical registers.
Evan Cheng97b5beb2012-01-10 02:02:58 +0000496 bool CrossMBBPhysDef = false;
Nick Lewycky7a7a6db2012-07-05 06:19:21 +0000497 SmallSet<unsigned, 8> PhysRefs;
Evan Cheng97b5beb2012-01-10 02:02:58 +0000498 SmallVector<unsigned, 2> PhysDefs;
Ulrich Weigandb64e2112012-11-13 18:40:58 +0000499 bool PhysUseDef = false;
500 if (FoundCSE && hasLivePhysRegDefUses(MI, MBB, PhysRefs,
501 PhysDefs, PhysUseDef)) {
Evan Cheng67bda722010-03-03 23:59:08 +0000502 FoundCSE = false;
503
Evan Cheng97b5beb2012-01-10 02:02:58 +0000504 // ... Unless the CS is local or is in the sole predecessor block
505 // and it also defines the physical register which is not clobbered
506 // in between and the physical register uses were not clobbered.
Ulrich Weigandb64e2112012-11-13 18:40:58 +0000507 // This can never be the case if the instruction both uses and
508 // defines the same physical register, which was detected above.
509 if (!PhysUseDef) {
510 unsigned CSVN = VNT.lookup(MI);
511 MachineInstr *CSMI = Exps[CSVN];
512 if (PhysRegDefsReach(CSMI, MI, PhysRefs, PhysDefs, CrossMBBPhysDef))
513 FoundCSE = true;
514 }
Evan Cheng835810b2010-05-21 21:22:19 +0000515 }
516
Evan Cheng16b48b82010-03-03 21:20:05 +0000517 if (!FoundCSE) {
518 VNT.insert(MI, CurrVN++);
519 Exps.push_back(MI);
520 continue;
521 }
522
523 // Found a common subexpression, eliminate it.
524 unsigned CSVN = VNT.lookup(MI);
525 MachineInstr *CSMI = Exps[CSVN];
526 DEBUG(dbgs() << "Examining: " << *MI);
527 DEBUG(dbgs() << "*** Found a common subexpression: " << *CSMI);
Evan Cheng31f94c72010-03-09 03:21:12 +0000528
529 // Check if it's profitable to perform this CSE.
530 bool DoCSE = true;
Manman Ren39ad5682012-08-08 00:51:41 +0000531 unsigned NumDefs = MI->getDesc().getNumDefs() +
532 MI->getDesc().getNumImplicitDefs();
Stephen Hines36b56882014-04-23 16:57:46 -0700533
Evan Cheng16b48b82010-03-03 21:20:05 +0000534 for (unsigned i = 0, e = MI->getNumOperands(); NumDefs && i != e; ++i) {
535 MachineOperand &MO = MI->getOperand(i);
536 if (!MO.isReg() || !MO.isDef())
537 continue;
538 unsigned OldReg = MO.getReg();
539 unsigned NewReg = CSMI->getOperand(i).getReg();
Manman Ren39ad5682012-08-08 00:51:41 +0000540
541 // Go through implicit defs of CSMI and MI, if a def is not dead at MI,
542 // we should make sure it is not dead at CSMI.
543 if (MO.isImplicit() && !MO.isDead() && CSMI->getOperand(i).isDead())
544 ImplicitDefsToUpdate.push_back(i);
545 if (OldReg == NewReg) {
546 --NumDefs;
Evan Cheng6cc1aea2010-03-06 01:14:19 +0000547 continue;
Manman Ren39ad5682012-08-08 00:51:41 +0000548 }
Bill Wendlingf6fb7ed2011-10-12 23:03:40 +0000549
Evan Cheng6cc1aea2010-03-06 01:14:19 +0000550 assert(TargetRegisterInfo::isVirtualRegister(OldReg) &&
Evan Cheng16b48b82010-03-03 21:20:05 +0000551 TargetRegisterInfo::isVirtualRegister(NewReg) &&
552 "Do not CSE physical register defs!");
Bill Wendlingf6fb7ed2011-10-12 23:03:40 +0000553
Evan Cheng2938a002010-03-10 02:12:03 +0000554 if (!isProfitableToCSE(NewReg, OldReg, CSMI, MI)) {
Nick Lewycky7a7a6db2012-07-05 06:19:21 +0000555 DEBUG(dbgs() << "*** Not profitable, avoid CSE!\n");
Evan Cheng31f94c72010-03-09 03:21:12 +0000556 DoCSE = false;
557 break;
558 }
Bill Wendlingf6fb7ed2011-10-12 23:03:40 +0000559
560 // Don't perform CSE if the result of the old instruction cannot exist
561 // within the register class of the new instruction.
562 const TargetRegisterClass *OldRC = MRI->getRegClass(OldReg);
563 if (!MRI->constrainRegClass(NewReg, OldRC)) {
Nick Lewycky7a7a6db2012-07-05 06:19:21 +0000564 DEBUG(dbgs() << "*** Not the same register class, avoid CSE!\n");
Bill Wendlingf6fb7ed2011-10-12 23:03:40 +0000565 DoCSE = false;
566 break;
567 }
568
Evan Cheng31f94c72010-03-09 03:21:12 +0000569 CSEPairs.push_back(std::make_pair(OldReg, NewReg));
Evan Cheng16b48b82010-03-03 21:20:05 +0000570 --NumDefs;
571 }
Evan Cheng31f94c72010-03-09 03:21:12 +0000572
573 // Actually perform the elimination.
574 if (DoCSE) {
Dan Gohman49b45892010-05-13 19:24:00 +0000575 for (unsigned i = 0, e = CSEPairs.size(); i != e; ++i) {
Evan Cheng31f94c72010-03-09 03:21:12 +0000576 MRI->replaceRegWith(CSEPairs[i].first, CSEPairs[i].second);
Dan Gohman49b45892010-05-13 19:24:00 +0000577 MRI->clearKillFlags(CSEPairs[i].second);
578 }
Evan Cheng97b5beb2012-01-10 02:02:58 +0000579
Manman Ren39ad5682012-08-08 00:51:41 +0000580 // Go through implicit defs of CSMI and MI, if a def is not dead at MI,
581 // we should make sure it is not dead at CSMI.
582 for (unsigned i = 0, e = ImplicitDefsToUpdate.size(); i != e; ++i)
583 CSMI->getOperand(ImplicitDefsToUpdate[i]).setIsDead(false);
584
Evan Cheng97b5beb2012-01-10 02:02:58 +0000585 if (CrossMBBPhysDef) {
586 // Add physical register defs now coming in from a predecessor to MBB
587 // livein list.
588 while (!PhysDefs.empty()) {
589 unsigned LiveIn = PhysDefs.pop_back_val();
590 if (!MBB->isLiveIn(LiveIn))
591 MBB->addLiveIn(LiveIn);
592 }
593 ++NumCrossBBCSEs;
594 }
595
Evan Cheng31f94c72010-03-09 03:21:12 +0000596 MI->eraseFromParent();
597 ++NumCSEs;
Evan Cheng189c1ec2010-10-29 23:36:03 +0000598 if (!PhysRefs.empty())
Evan Cheng2b4e7272010-06-04 23:28:13 +0000599 ++NumPhysCSEs;
Evan Chenga63cde22010-12-15 22:16:21 +0000600 if (Commuted)
601 ++NumCommutes;
Evan Chengcfea9852011-04-11 18:47:20 +0000602 Changed = true;
Evan Cheng31f94c72010-03-09 03:21:12 +0000603 } else {
Evan Cheng31f94c72010-03-09 03:21:12 +0000604 VNT.insert(MI, CurrVN++);
605 Exps.push_back(MI);
606 }
607 CSEPairs.clear();
Manman Ren39ad5682012-08-08 00:51:41 +0000608 ImplicitDefsToUpdate.clear();
Evan Cheng6ba95542010-03-03 02:48:20 +0000609 }
610
Evan Cheng31156982010-04-21 00:21:07 +0000611 return Changed;
612}
613
614/// ExitScopeIfDone - Destroy scope for the MBB that corresponds to the given
615/// dominator tree node if its a leaf or all of its children are done. Walk
616/// up the dominator tree to destroy ancestors which are now done.
617void
618MachineCSE::ExitScopeIfDone(MachineDomTreeNode *Node,
Nick Lewycky7a7a6db2012-07-05 06:19:21 +0000619 DenseMap<MachineDomTreeNode*, unsigned> &OpenChildren) {
Evan Cheng31156982010-04-21 00:21:07 +0000620 if (OpenChildren[Node])
621 return;
622
623 // Pop scope.
624 ExitScope(Node->getBlock());
625
626 // Now traverse upwards to pop ancestors whose offsprings are all done.
Nick Lewycky7a7a6db2012-07-05 06:19:21 +0000627 while (MachineDomTreeNode *Parent = Node->getIDom()) {
Evan Cheng31156982010-04-21 00:21:07 +0000628 unsigned Left = --OpenChildren[Parent];
629 if (Left != 0)
630 break;
631 ExitScope(Parent->getBlock());
632 Node = Parent;
633 }
634}
635
636bool MachineCSE::PerformCSE(MachineDomTreeNode *Node) {
637 SmallVector<MachineDomTreeNode*, 32> Scopes;
638 SmallVector<MachineDomTreeNode*, 8> WorkList;
Evan Cheng31156982010-04-21 00:21:07 +0000639 DenseMap<MachineDomTreeNode*, unsigned> OpenChildren;
640
Evan Chengc2b768f2010-09-17 21:59:42 +0000641 CurrVN = 0;
642
Evan Cheng31156982010-04-21 00:21:07 +0000643 // Perform a DFS walk to determine the order of visit.
644 WorkList.push_back(Node);
645 do {
646 Node = WorkList.pop_back_val();
647 Scopes.push_back(Node);
648 const std::vector<MachineDomTreeNode*> &Children = Node->getChildren();
649 unsigned NumChildren = Children.size();
650 OpenChildren[Node] = NumChildren;
651 for (unsigned i = 0; i != NumChildren; ++i) {
652 MachineDomTreeNode *Child = Children[i];
Evan Cheng31156982010-04-21 00:21:07 +0000653 WorkList.push_back(Child);
654 }
655 } while (!WorkList.empty());
656
657 // Now perform CSE.
658 bool Changed = false;
659 for (unsigned i = 0, e = Scopes.size(); i != e; ++i) {
660 MachineDomTreeNode *Node = Scopes[i];
661 MachineBasicBlock *MBB = Node->getBlock();
662 EnterScope(MBB);
663 Changed |= ProcessBlock(MBB);
664 // If it's a leaf node, it's done. Traverse upwards to pop ancestors.
Nick Lewycky7a7a6db2012-07-05 06:19:21 +0000665 ExitScopeIfDone(Node, OpenChildren);
Evan Cheng31156982010-04-21 00:21:07 +0000666 }
Evan Cheng6ba95542010-03-03 02:48:20 +0000667
668 return Changed;
669}
670
Evan Chengc6fe3332010-03-02 02:38:24 +0000671bool MachineCSE::runOnMachineFunction(MachineFunction &MF) {
Stephen Hines36b56882014-04-23 16:57:46 -0700672 if (skipOptnoneFunction(*MF.getFunction()))
673 return false;
674
Stephen Hines37ed9c12014-12-01 14:51:49 -0800675 TII = MF.getSubtarget().getInstrInfo();
676 TRI = MF.getSubtarget().getRegisterInfo();
Evan Cheng6ba95542010-03-03 02:48:20 +0000677 MRI = &MF.getRegInfo();
Evan Chenga5f32cb2010-03-04 21:18:08 +0000678 AA = &getAnalysis<AliasAnalysis>();
Evan Cheng31f94c72010-03-09 03:21:12 +0000679 DT = &getAnalysis<MachineDominatorTree>();
Evan Cheng31156982010-04-21 00:21:07 +0000680 return PerformCSE(DT->getRootNode());
Evan Chengc6fe3332010-03-02 02:38:24 +0000681}