Quentin Colombet | 8e8e85c | 2016-04-05 19:06:01 +0000 | [diff] [blame] | 1 | //===- llvm/CodeGen/GlobalISel/RegBankSelect.cpp - RegBankSelect -*- C++ -*-==// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | /// \file |
| 10 | /// This file implements the RegBankSelect class. |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" |
Quentin Colombet | cfd97b9 | 2016-05-20 00:35:26 +0000 | [diff] [blame] | 14 | #include "llvm/ADT/PostOrderIterator.h" |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 15 | #include "llvm/CodeGen/GlobalISel/RegisterBank.h" |
Quentin Colombet | 5565075 | 2016-05-20 00:49:10 +0000 | [diff] [blame^] | 16 | #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" |
| 17 | #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 18 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
Quentin Colombet | cfd97b9 | 2016-05-20 00:35:26 +0000 | [diff] [blame] | 19 | #include "llvm/Support/BlockFrequency.h" |
Quentin Colombet | e16f561 | 2016-04-07 23:53:55 +0000 | [diff] [blame] | 20 | #include "llvm/Support/Debug.h" |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 21 | #include "llvm/Target/TargetSubtargetInfo.h" |
Quentin Colombet | 8e8e85c | 2016-04-05 19:06:01 +0000 | [diff] [blame] | 22 | |
| 23 | #define DEBUG_TYPE "regbankselect" |
| 24 | |
| 25 | using namespace llvm; |
| 26 | |
| 27 | char RegBankSelect::ID = 0; |
| 28 | INITIALIZE_PASS(RegBankSelect, "regbankselect", |
| 29 | "Assign register bank of generic virtual registers", |
| 30 | false, false); |
| 31 | |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 32 | RegBankSelect::RegBankSelect() |
| 33 | : MachineFunctionPass(ID), RBI(nullptr), MRI(nullptr) { |
Quentin Colombet | 8e8e85c | 2016-04-05 19:06:01 +0000 | [diff] [blame] | 34 | initializeRegBankSelectPass(*PassRegistry::getPassRegistry()); |
| 35 | } |
| 36 | |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 37 | void RegBankSelect::init(MachineFunction &MF) { |
| 38 | RBI = MF.getSubtarget().getRegBankInfo(); |
| 39 | assert(RBI && "Cannot work without RegisterBankInfo"); |
| 40 | MRI = &MF.getRegInfo(); |
Quentin Colombet | aac71a4 | 2016-04-07 21:32:23 +0000 | [diff] [blame] | 41 | TRI = MF.getSubtarget().getRegisterInfo(); |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 42 | MIRBuilder.setMF(MF); |
| 43 | } |
| 44 | |
| 45 | bool RegBankSelect::assignmentMatch( |
Quentin Colombet | 0d77da4 | 2016-05-20 00:42:57 +0000 | [diff] [blame] | 46 | unsigned Reg, const RegisterBankInfo::ValueMapping &ValMapping, |
| 47 | bool &OnlyAssign) const { |
| 48 | // By default we assume we will have to repair something. |
| 49 | OnlyAssign = false; |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 50 | // Each part of a break down needs to end up in a different register. |
| 51 | // In other word, Reg assignement does not match. |
| 52 | if (ValMapping.BreakDown.size() > 1) |
| 53 | return false; |
| 54 | |
Quentin Colombet | 6d6d6af | 2016-04-08 16:48:16 +0000 | [diff] [blame] | 55 | const RegisterBank *CurRegBank = RBI->getRegBank(Reg, *MRI, *TRI); |
| 56 | const RegisterBank *DesiredRegBrank = ValMapping.BreakDown[0].RegBank; |
Quentin Colombet | 0d77da4 | 2016-05-20 00:42:57 +0000 | [diff] [blame] | 57 | // Reg is free of assignment, a simple assignment will make the |
| 58 | // register bank to match. |
| 59 | OnlyAssign = CurRegBank == nullptr; |
Quentin Colombet | 6d6d6af | 2016-04-08 16:48:16 +0000 | [diff] [blame] | 60 | DEBUG(dbgs() << "Does assignment already match: "; |
| 61 | if (CurRegBank) dbgs() << *CurRegBank; else dbgs() << "none"; |
| 62 | dbgs() << " against "; |
| 63 | assert(DesiredRegBrank && "The mapping must be valid"); |
| 64 | dbgs() << *DesiredRegBrank << '\n';); |
| 65 | return CurRegBank == DesiredRegBrank; |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 66 | } |
| 67 | |
| 68 | unsigned |
| 69 | RegBankSelect::repairReg(unsigned Reg, |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 70 | const RegisterBankInfo::ValueMapping &ValMapping, |
| 71 | MachineInstr &DefUseMI, bool IsDef) { |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 72 | assert(ValMapping.BreakDown.size() == 1 && |
| 73 | "Support for complex break down not supported yet"); |
| 74 | const RegisterBankInfo::PartialMapping &PartialMap = ValMapping.BreakDown[0]; |
Quentin Colombet | 0e5ff58 | 2016-04-21 18:09:34 +0000 | [diff] [blame] | 75 | assert(PartialMap.Length == |
Quentin Colombet | 777a771 | 2016-04-12 00:38:51 +0000 | [diff] [blame] | 76 | (TargetRegisterInfo::isPhysicalRegister(Reg) |
| 77 | ? TRI->getMinimalPhysRegClass(Reg)->getSize() * 8 |
| 78 | : MRI->getSize(Reg)) && |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 79 | "Repairing other than copy not implemented yet"); |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 80 | // If the MIRBuilder is configured to insert somewhere else than |
| 81 | // DefUseMI, we may not use this function like was it first |
| 82 | // internded (local repairing), so make sure we pay attention before |
| 83 | // we remove the assert. |
| 84 | // In particular, it is likely that we will have to properly save |
| 85 | // the insertion point of the MIRBuilder and restore it at the end |
| 86 | // of this method. |
| 87 | assert(&DefUseMI == &(*MIRBuilder.getInsertPt()) && |
| 88 | "Need to save and restore the insertion point"); |
| 89 | // For use, we will add a copy just in front of the instruction. |
| 90 | // For def, we will add a copy just after the instruction. |
| 91 | // In either case, the insertion point must be valid. In particular, |
| 92 | // make sure we do not insert in the middle of terminators or phis. |
| 93 | bool Before = !IsDef; |
| 94 | setSafeInsertionPoint(DefUseMI, Before); |
| 95 | if (DefUseMI.isTerminator() && Before) { |
| 96 | // Check that the insertion point does not happen |
| 97 | // before the definition of Reg. |
| 98 | // This can happen if Reg is defined by a terminator |
| 99 | // and used by another one. |
| 100 | // In that case the repairing code is actually more involved |
| 101 | // because we have to split the block. |
| 102 | |
| 103 | // Assert that this is not a physical register. |
| 104 | // The target independent code does not insert physical registers |
| 105 | // on terminators, so if we end up in this situation, this is |
| 106 | // likely a bug in the target. |
| 107 | assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && |
| 108 | "Check for physical register not implemented"); |
| 109 | const MachineInstr *RegDef = MRI->getVRegDef(Reg); |
| 110 | assert(RegDef && "Reg has more than one definition?"); |
| 111 | // Assert to make the code more readable; Reg is used by DefUseMI, i.e., |
| 112 | // (Before == !IsDef == true), so DefUseMI != RegDef otherwise we have |
| 113 | // a use (that is not a PHI) that is not dominated by its def. |
| 114 | assert(&DefUseMI != RegDef && "Def does not dominate all of its uses"); |
| 115 | if (RegDef->isTerminator() && RegDef->getParent() == DefUseMI.getParent()) |
| 116 | // By construction, the repairing should happen between two |
| 117 | // terminators: RegDef and DefUseMI. |
| 118 | // This is not implemented. |
| 119 | report_fatal_error("Repairing between terminators not implemented yet"); |
| 120 | } |
| 121 | |
| 122 | // Create a new temporary to hold the repaired value. |
Quentin Colombet | 0e5ff58 | 2016-04-21 18:09:34 +0000 | [diff] [blame] | 123 | unsigned NewReg = MRI->createGenericVirtualRegister(PartialMap.Length); |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 124 | // Set the registers for the source and destination of the copy. |
| 125 | unsigned Src = Reg, Dst = NewReg; |
| 126 | // If this is a definition that we repair, the copy will be |
| 127 | // inverted. |
| 128 | if (IsDef) |
| 129 | std::swap(Src, Dst); |
| 130 | (void)MIRBuilder.buildInstr(TargetOpcode::COPY, Dst, Src); |
| 131 | |
Quentin Colombet | e16f561 | 2016-04-07 23:53:55 +0000 | [diff] [blame] | 132 | DEBUG(dbgs() << "Repair: " << PrintReg(Reg) << " with: " |
| 133 | << PrintReg(NewReg) << '\n'); |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 134 | |
| 135 | // Restore the insertion point of the MIRBuilder. |
| 136 | MIRBuilder.setInstr(DefUseMI, Before); |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 137 | return NewReg; |
| 138 | } |
| 139 | |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 140 | void RegBankSelect::setSafeInsertionPoint(MachineInstr &InsertPt, bool Before) { |
| 141 | // Check that we are not looking to insert before a phi. |
| 142 | // Indeed, we would need more information on what to do. |
| 143 | // By default that should be all the predecessors, but this is |
| 144 | // probably not what we want in general. |
| 145 | assert((!Before || !InsertPt.isPHI()) && |
| 146 | "Insertion before phis not implemented"); |
| 147 | // The same kind of observation hold for terminators if we try to |
| 148 | // insert after them. |
| 149 | assert((Before || !InsertPt.isTerminator()) && |
| 150 | "Insertion after terminatos not implemented"); |
| 151 | if (InsertPt.isPHI()) { |
| 152 | assert(!Before && "Not supported!!"); |
| 153 | MachineBasicBlock *MBB = InsertPt.getParent(); |
| 154 | assert(MBB && "Insertion point is not in a basic block"); |
| 155 | MachineBasicBlock::iterator FirstNonPHIPt = MBB->getFirstNonPHI(); |
| 156 | if (FirstNonPHIPt == MBB->end()) { |
| 157 | // If there is not any non-phi instruction, insert at the end of MBB. |
| 158 | MIRBuilder.setMBB(*MBB, /*Beginning*/ false); |
| 159 | return; |
| 160 | } |
| 161 | // The insertion point before the first non-phi instruction. |
| 162 | MIRBuilder.setInstr(*FirstNonPHIPt, /*Before*/ true); |
| 163 | return; |
| 164 | } |
| 165 | if (InsertPt.isTerminator()) { |
| 166 | MachineBasicBlock *MBB = InsertPt.getParent(); |
| 167 | assert(MBB && "Insertion point is not in a basic block"); |
| 168 | MIRBuilder.setInstr(*MBB->getFirstTerminator(), /*Before*/ true); |
| 169 | return; |
| 170 | } |
| 171 | MIRBuilder.setInstr(InsertPt, /*Before*/ Before); |
| 172 | } |
| 173 | |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 174 | void RegBankSelect::assignInstr(MachineInstr &MI) { |
Quentin Colombet | e16f561 | 2016-04-07 23:53:55 +0000 | [diff] [blame] | 175 | DEBUG(dbgs() << "Assign: " << MI); |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 176 | const RegisterBankInfo::InstructionMapping DefaultMapping = |
| 177 | RBI->getInstrMapping(MI); |
| 178 | // Make sure the mapping is valid for MI. |
Quentin Colombet | c320fb4 | 2016-04-21 18:34:43 +0000 | [diff] [blame] | 179 | assert(DefaultMapping.verify(MI) && "Invalid instruction mapping"); |
Quentin Colombet | e16f561 | 2016-04-07 23:53:55 +0000 | [diff] [blame] | 180 | |
| 181 | DEBUG(dbgs() << "Mapping: " << DefaultMapping << '\n'); |
| 182 | |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 183 | // Set the insertion point before MI. |
| 184 | // This is where we are going to insert the repairing code if any. |
| 185 | MIRBuilder.setInstr(MI, /*Before*/ true); |
| 186 | |
| 187 | // For now, do not look for alternative mappings. |
| 188 | // Alternative mapping may require to rewrite MI and we do not support |
| 189 | // that yet. |
| 190 | // Walk the operands and assign then to the chosen mapping, possibly with |
| 191 | // the insertion of repair code for uses. |
| 192 | for (unsigned OpIdx = 0, EndIdx = MI.getNumOperands(); OpIdx != EndIdx; |
| 193 | ++OpIdx) { |
| 194 | MachineOperand &MO = MI.getOperand(OpIdx); |
| 195 | // Nothing to be done for non-register operands. |
| 196 | if (!MO.isReg()) |
| 197 | continue; |
| 198 | unsigned Reg = MO.getReg(); |
| 199 | if (!Reg) |
| 200 | continue; |
| 201 | |
| 202 | const RegisterBankInfo::ValueMapping &ValMapping = |
| 203 | DefaultMapping.getOperandMapping(OpIdx); |
| 204 | // If Reg is already properly mapped, move on. |
Quentin Colombet | 0d77da4 | 2016-05-20 00:42:57 +0000 | [diff] [blame] | 205 | bool OnlyAssign; |
| 206 | if (assignmentMatch(Reg, ValMapping, OnlyAssign)) |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 207 | continue; |
| 208 | |
| 209 | // For uses, we may need to create a new temporary. |
| 210 | // Indeed, if Reg is already assigned a register bank, at this |
| 211 | // point, we know it is different from the one defined by the |
| 212 | // chosen mapping, we need to adjust for that. |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 213 | // For definitions, changing the register bank will affect all |
| 214 | // its uses, and in particular the ones we already visited. |
| 215 | // Although this is correct, since with the RPO traversal of the |
| 216 | // basic blocks the only uses that we already visisted for this |
| 217 | // definition are PHIs (i.e., copies), this may not be the best |
| 218 | // solution according to the cost model. |
| 219 | // Therefore, create a new temporary for Reg. |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 220 | assert(ValMapping.BreakDown.size() == 1 && |
| 221 | "Support for complex break down not supported yet"); |
Quentin Colombet | 0d77da4 | 2016-05-20 00:42:57 +0000 | [diff] [blame] | 222 | if (!OnlyAssign) { |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 223 | if (!MO.isDef() && MI.isPHI()) { |
| 224 | // Phis are already copies, so there is nothing to repair. |
| 225 | // Note: This will not hold when we support break downs with |
| 226 | // more than one segment. |
| 227 | DEBUG(dbgs() << "Skip PHI use\n"); |
| 228 | continue; |
| 229 | } |
| 230 | // If MO is a definition, since repairing after a terminator is |
| 231 | // painful, do not repair. Indeed, this is probably not worse |
| 232 | // saving the move in the PHIs that will get reassigned. |
| 233 | if (!MO.isDef() || !MI.isTerminator()) |
| 234 | Reg = repairReg(Reg, ValMapping, MI, MO.isDef()); |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 235 | } |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 236 | |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 237 | // If we end up here, MO should be free of encoding constraints, |
| 238 | // i.e., we do not have to constrained the RegBank of Reg to |
| 239 | // the requirement of the operands. |
| 240 | // If that is not the case, this means the code was broken before |
| 241 | // hands because we should have found that the assignment match. |
| 242 | // This will not hold when we will consider alternative mappings. |
Quentin Colombet | 6d6d6af | 2016-04-08 16:48:16 +0000 | [diff] [blame] | 243 | DEBUG(dbgs() << "Assign: " << *ValMapping.BreakDown[0].RegBank << " to " |
| 244 | << PrintReg(Reg) << '\n'); |
Quentin Colombet | 904a2c7 | 2016-04-12 00:12:59 +0000 | [diff] [blame] | 245 | |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 246 | MRI->setRegBank(Reg, *ValMapping.BreakDown[0].RegBank); |
| 247 | MO.setReg(Reg); |
| 248 | } |
Quentin Colombet | e16f561 | 2016-04-07 23:53:55 +0000 | [diff] [blame] | 249 | DEBUG(dbgs() << "Assigned: " << MI); |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 250 | } |
| 251 | |
Quentin Colombet | 8e8e85c | 2016-04-05 19:06:01 +0000 | [diff] [blame] | 252 | bool RegBankSelect::runOnMachineFunction(MachineFunction &MF) { |
Quentin Colombet | e16f561 | 2016-04-07 23:53:55 +0000 | [diff] [blame] | 253 | DEBUG(dbgs() << "Assign register banks for: " << MF.getName() << '\n'); |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 254 | init(MF); |
| 255 | // Walk the function and assign register banks to all operands. |
Quentin Colombet | ab8c21f | 2016-04-08 17:19:10 +0000 | [diff] [blame] | 256 | // Use a RPOT to make sure all registers are assigned before we choose |
| 257 | // the best mapping of the current instruction. |
| 258 | ReversePostOrderTraversal<MachineFunction*> RPOT(&MF); |
| 259 | for (MachineBasicBlock *MBB : RPOT) |
| 260 | for (MachineInstr &MI : *MBB) |
Quentin Colombet | 40ad573 | 2016-04-07 18:19:27 +0000 | [diff] [blame] | 261 | assignInstr(MI); |
Quentin Colombet | 8e8e85c | 2016-04-05 19:06:01 +0000 | [diff] [blame] | 262 | return false; |
| 263 | } |
Quentin Colombet | cfd97b9 | 2016-05-20 00:35:26 +0000 | [diff] [blame] | 264 | |
| 265 | //------------------------------------------------------------------------------ |
Quentin Colombet | 5565075 | 2016-05-20 00:49:10 +0000 | [diff] [blame^] | 266 | // Helper Classes Implementation |
Quentin Colombet | cfd97b9 | 2016-05-20 00:35:26 +0000 | [diff] [blame] | 267 | //------------------------------------------------------------------------------ |
Quentin Colombet | 5565075 | 2016-05-20 00:49:10 +0000 | [diff] [blame^] | 268 | RegBankSelect::RepairingPlacement::RepairingPlacement( |
| 269 | MachineInstr &MI, unsigned OpIdx, const TargetRegisterInfo &TRI, Pass &P, |
| 270 | RepairingPlacement::RepairingKind Kind) |
| 271 | // Default is, we are going to insert code to repair OpIdx. |
| 272 | : Kind(Kind), |
| 273 | OpIdx(OpIdx), |
| 274 | CanMaterialize(Kind != RepairingKind::Impossible), |
| 275 | HasSplit(false), |
| 276 | P(P) { |
| 277 | const MachineOperand &MO = MI.getOperand(OpIdx); |
| 278 | assert(MO.isReg() && "Trying to repair a non-reg operand"); |
| 279 | |
| 280 | if (Kind != RepairingKind::Insert) |
| 281 | return; |
| 282 | |
| 283 | // Repairings for definitions happen after MI, uses happen before. |
| 284 | bool Before = !MO.isDef(); |
| 285 | |
| 286 | // Check if we are done with MI. |
| 287 | if (!MI.isPHI() && !MI.isTerminator()) { |
| 288 | addInsertPoint(MI, Before); |
| 289 | // We are done with the initialization. |
| 290 | return; |
| 291 | } |
| 292 | |
| 293 | // Now, look for the special cases. |
| 294 | if (MI.isPHI()) { |
| 295 | // - PHI must be the first instructions: |
| 296 | // * Before, we have to split the related incoming edge. |
| 297 | // * After, move the insertion point past the last phi. |
| 298 | if (!Before) { |
| 299 | MachineBasicBlock::iterator It = MI.getParent()->getFirstNonPHI(); |
| 300 | if (It != MI.getParent()->end()) |
| 301 | addInsertPoint(*It, /*Before*/ true); |
| 302 | else |
| 303 | addInsertPoint(*(--It), /*Before*/ false); |
| 304 | return; |
| 305 | } |
| 306 | // We repair a use of a phi, we may need to split the related edge. |
| 307 | MachineBasicBlock &Pred = *MI.getOperand(OpIdx + 1).getMBB(); |
| 308 | // Check if we can move the insertion point prior to the |
| 309 | // terminators of the predecessor. |
| 310 | unsigned Reg = MO.getReg(); |
| 311 | MachineBasicBlock::iterator It = Pred.getLastNonDebugInstr(); |
| 312 | for (auto Begin = Pred.begin(); It != Begin && It->isTerminator(); --It) |
| 313 | if (It->modifiesRegister(Reg, &TRI)) { |
| 314 | // We cannot hoist the repairing code in the predecessor. |
| 315 | // Split the edge. |
| 316 | addInsertPoint(Pred, *MI.getParent()); |
| 317 | return; |
| 318 | } |
| 319 | // At this point, we can insert in Pred. |
| 320 | |
| 321 | // - If It is invalid, Pred is empty and we can insert in Pred |
| 322 | // wherever we want. |
| 323 | // - If It is valid, It is the first non-terminator, insert after It. |
| 324 | if (It == Pred.end()) |
| 325 | addInsertPoint(Pred, /*Beginning*/ false); |
| 326 | else |
| 327 | addInsertPoint(*It, /*Before*/ false); |
| 328 | } else { |
| 329 | // - Terminators must be the last instructions: |
| 330 | // * Before, move the insert point before the first terminator. |
| 331 | // * After, we have to split the outcoming edges. |
| 332 | unsigned Reg = MO.getReg(); |
| 333 | if (Before) { |
| 334 | // Check whether Reg is defined by any terminator. |
| 335 | MachineBasicBlock::iterator It = MI; |
| 336 | for (auto Begin = MI.getParent()->begin(); |
| 337 | --It != Begin && It->isTerminator();) |
| 338 | if (It->modifiesRegister(Reg, &TRI)) { |
| 339 | // Insert the repairing code right after the definition. |
| 340 | addInsertPoint(*It, /*Before*/ false); |
| 341 | return; |
| 342 | } |
| 343 | addInsertPoint(*It, /*Before*/ true); |
| 344 | return; |
| 345 | } |
| 346 | // Make sure Reg is not redefined by other terminators, otherwise |
| 347 | // we do not know how to split. |
| 348 | for (MachineBasicBlock::iterator It = MI, End = MI.getParent()->end(); |
| 349 | ++It != End;) |
| 350 | // The machine verifier should reject this kind of code. |
| 351 | assert(It->modifiesRegister(Reg, &TRI) && "Do not know where to split"); |
| 352 | // Split each outcoming edges. |
| 353 | MachineBasicBlock &Src = *MI.getParent(); |
| 354 | for (auto &Succ : Src.successors()) |
| 355 | addInsertPoint(Src, Succ); |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | void RegBankSelect::RepairingPlacement::addInsertPoint(MachineInstr &MI, |
| 360 | bool Before) { |
| 361 | addInsertPoint(*new InstrInsertPoint(MI, Before)); |
| 362 | } |
| 363 | |
| 364 | void RegBankSelect::RepairingPlacement::addInsertPoint(MachineBasicBlock &MBB, |
| 365 | bool Beginning) { |
| 366 | addInsertPoint(*new MBBInsertPoint(MBB, Beginning)); |
| 367 | } |
| 368 | |
| 369 | void RegBankSelect::RepairingPlacement::addInsertPoint(MachineBasicBlock &Src, |
| 370 | MachineBasicBlock &Dst) { |
| 371 | addInsertPoint(*new EdgeInsertPoint(Src, Dst, P)); |
| 372 | } |
| 373 | |
| 374 | void RegBankSelect::RepairingPlacement::addInsertPoint( |
| 375 | RegBankSelect::InsertPoint &Point) { |
| 376 | CanMaterialize &= Point.canMaterialize(); |
| 377 | HasSplit |= Point.isSplit(); |
| 378 | InsertPoints.emplace_back(&Point); |
| 379 | } |
| 380 | |
| 381 | RegBankSelect::InstrInsertPoint::InstrInsertPoint(MachineInstr &Instr, |
| 382 | bool Before) |
| 383 | : InsertPoint(), Instr(Instr), Before(Before) { |
| 384 | // Since we do not support splitting, we do not need to update |
| 385 | // liveness and such, so do not do anything with P. |
| 386 | assert((!Before || !Instr.isPHI()) && |
| 387 | "Splitting before phis requires more points"); |
| 388 | assert((!Before || !Instr.getNextNode() || !Instr.getNextNode()->isPHI()) && |
| 389 | "Splitting between phis does not make sense"); |
| 390 | } |
| 391 | |
| 392 | void RegBankSelect::InstrInsertPoint::materialize() { |
| 393 | if (isSplit()) { |
| 394 | // Slice and return the beginning of the new block. |
| 395 | // If we need to split between the terminators, we theoritically |
| 396 | // need to know where the first and second set of terminators end |
| 397 | // to update the successors properly. |
| 398 | // Now, in pratice, we should have a maximum of 2 branch |
| 399 | // instructions; one conditional and one unconditional. Therefore |
| 400 | // we know how to update the successor by looking at the target of |
| 401 | // the unconditional branch. |
| 402 | // If we end up splitting at some point, then, we should update |
| 403 | // the liveness information and such. I.e., we would need to |
| 404 | // access P here. |
| 405 | // The machine verifier should actually make sure such cases |
| 406 | // cannot happen. |
| 407 | llvm_unreachable("Not yet implemented"); |
| 408 | } |
| 409 | // Otherwise the insertion point is just the current or next |
| 410 | // instruction depending on Before. I.e., there is nothing to do |
| 411 | // here. |
| 412 | } |
| 413 | |
| 414 | bool RegBankSelect::InstrInsertPoint::isSplit() const { |
| 415 | // If the insertion point is after a terminator, we need to split. |
| 416 | if (!Before) |
| 417 | return Instr.isTerminator(); |
| 418 | // If we insert before an instruction that is after a terminator, |
| 419 | // we are still after a terminator. |
| 420 | return Instr.getPrevNode() && Instr.getPrevNode()->isTerminator(); |
| 421 | } |
| 422 | |
| 423 | uint64_t RegBankSelect::InstrInsertPoint::frequency(const Pass &P) const { |
| 424 | // Even if we need to split, because we insert between terminators, |
| 425 | // this split has actually the same frequency as the instruction. |
| 426 | const MachineBlockFrequencyInfo *MBFI = |
| 427 | P.getAnalysisIfAvailable<MachineBlockFrequencyInfo>(); |
| 428 | if (!MBFI) |
| 429 | return 1; |
| 430 | return MBFI->getBlockFreq(Instr.getParent()).getFrequency(); |
| 431 | } |
| 432 | |
| 433 | uint64_t RegBankSelect::MBBInsertPoint::frequency(const Pass &P) const { |
| 434 | const MachineBlockFrequencyInfo *MBFI = |
| 435 | P.getAnalysisIfAvailable<MachineBlockFrequencyInfo>(); |
| 436 | if (!MBFI) |
| 437 | return 1; |
| 438 | return MBFI->getBlockFreq(&MBB).getFrequency(); |
| 439 | } |
| 440 | |
| 441 | void RegBankSelect::EdgeInsertPoint::materialize() { |
| 442 | // If we end up repairing twice at the same place before materializing the |
| 443 | // insertion point, we may think we have to split an edge twice. |
| 444 | // We should have a factory for the insert point such that identical points |
| 445 | // are the same instance. |
| 446 | assert(Src.isSuccessor(DstOrSplit) && DstOrSplit->isPredecessor(&Src) && |
| 447 | "This point has already been split"); |
| 448 | MachineBasicBlock *NewBB = Src.SplitCriticalEdge(DstOrSplit, P); |
| 449 | assert(NewBB && "Invalid call to materialize"); |
| 450 | // We reuse the destination block to hold the information of the new block. |
| 451 | DstOrSplit = NewBB; |
| 452 | } |
| 453 | |
| 454 | uint64_t RegBankSelect::EdgeInsertPoint::frequency(const Pass &P) const { |
| 455 | const MachineBlockFrequencyInfo *MBFI = |
| 456 | P.getAnalysisIfAvailable<MachineBlockFrequencyInfo>(); |
| 457 | if (!MBFI) |
| 458 | return 1; |
| 459 | if (WasMaterialized) |
| 460 | return MBFI->getBlockFreq(DstOrSplit).getFrequency(); |
| 461 | |
| 462 | const MachineBranchProbabilityInfo *MBPI = |
| 463 | P.getAnalysisIfAvailable<MachineBranchProbabilityInfo>(); |
| 464 | if (!MBPI) |
| 465 | return 1; |
| 466 | // The basic block will be on the edge. |
| 467 | return (MBFI->getBlockFreq(&Src) * MBPI->getEdgeProbability(&Src, DstOrSplit)) |
| 468 | .getFrequency(); |
| 469 | } |
| 470 | |
| 471 | bool RegBankSelect::EdgeInsertPoint::canMaterialize() const { |
| 472 | // If this is not a critical edge, we should not have used this insert |
| 473 | // point. Indeed, either the successor or the predecessor should |
| 474 | // have do. |
| 475 | assert(Src.succ_size() > 1 && DstOrSplit->pred_size() > 1 && |
| 476 | "Edge is not critical"); |
| 477 | return Src.canSplitCriticalEdge(DstOrSplit); |
| 478 | } |
| 479 | |
Quentin Colombet | cfd97b9 | 2016-05-20 00:35:26 +0000 | [diff] [blame] | 480 | RegBankSelect::MappingCost::MappingCost(const BlockFrequency &LocalFreq) |
| 481 | : LocalCost(0), NonLocalCost(0), LocalFreq(LocalFreq.getFrequency()) {} |
| 482 | |
| 483 | bool RegBankSelect::MappingCost::addLocalCost(uint64_t Cost) { |
| 484 | // Check if this overflows. |
| 485 | if (LocalCost + Cost < LocalCost) { |
| 486 | saturate(); |
| 487 | return true; |
| 488 | } |
| 489 | LocalCost += Cost; |
| 490 | return isSaturated(); |
| 491 | } |
| 492 | |
| 493 | bool RegBankSelect::MappingCost::addNonLocalCost(uint64_t Cost) { |
| 494 | // Check if this overflows. |
| 495 | if (NonLocalCost + Cost < NonLocalCost) { |
| 496 | saturate(); |
| 497 | return true; |
| 498 | } |
| 499 | NonLocalCost += Cost; |
| 500 | return isSaturated(); |
| 501 | } |
| 502 | |
| 503 | bool RegBankSelect::MappingCost::isSaturated() const { |
| 504 | return LocalCost == UINT64_MAX - 1 && NonLocalCost == UINT64_MAX && |
| 505 | LocalFreq == UINT64_MAX; |
| 506 | } |
| 507 | |
| 508 | void RegBankSelect::MappingCost::saturate() { |
| 509 | *this = ImpossibleCost(); |
| 510 | --LocalCost; |
| 511 | } |
| 512 | |
| 513 | RegBankSelect::MappingCost RegBankSelect::MappingCost::ImpossibleCost() { |
| 514 | return MappingCost(UINT64_MAX, UINT64_MAX, UINT64_MAX); |
| 515 | } |
| 516 | |
| 517 | bool RegBankSelect::MappingCost::operator<(const MappingCost &Cost) const { |
| 518 | // Sort out the easy cases. |
| 519 | if (*this == Cost) |
| 520 | return false; |
| 521 | // If one is impossible to realize the other is cheaper unless it is |
| 522 | // impossible as well. |
| 523 | if ((*this == ImpossibleCost()) || (Cost == ImpossibleCost())) |
| 524 | return (*this == ImpossibleCost()) < (Cost == ImpossibleCost()); |
| 525 | // If one is saturated the other is cheaper, unless it is saturated |
| 526 | // as well. |
| 527 | if (isSaturated() || Cost.isSaturated()) |
| 528 | return isSaturated() < Cost.isSaturated(); |
| 529 | // At this point we know both costs hold sensible values. |
| 530 | |
| 531 | // If both values have a different base frequency, there is no much |
| 532 | // we can do but to scale everything. |
| 533 | // However, if they have the same base frequency we can avoid making |
| 534 | // complicated computation. |
| 535 | uint64_t ThisLocalAdjust; |
| 536 | uint64_t OtherLocalAdjust; |
| 537 | if (LLVM_LIKELY(LocalFreq == Cost.LocalFreq)) { |
| 538 | |
| 539 | // At this point, we know the local costs are comparable. |
| 540 | // Do the case that do not involve potential overflow first. |
| 541 | if (NonLocalCost == Cost.NonLocalCost) |
| 542 | // Since the non-local costs do not discriminate on the result, |
| 543 | // just compare the local costs. |
| 544 | return LocalCost < Cost.LocalCost; |
| 545 | |
| 546 | // The base costs are comparable so we may only keep the relative |
| 547 | // value to increase our chances of avoiding overflows. |
| 548 | ThisLocalAdjust = 0; |
| 549 | OtherLocalAdjust = 0; |
| 550 | if (LocalCost < Cost.LocalCost) |
| 551 | OtherLocalAdjust = Cost.LocalCost - LocalCost; |
| 552 | else |
| 553 | ThisLocalAdjust = LocalCost - Cost.LocalCost; |
| 554 | |
| 555 | } else { |
| 556 | ThisLocalAdjust = LocalCost; |
| 557 | OtherLocalAdjust = Cost.LocalCost; |
| 558 | } |
| 559 | |
| 560 | // The non-local costs are comparable, just keep the relative value. |
| 561 | uint64_t ThisNonLocalAdjust = 0; |
| 562 | uint64_t OtherNonLocalAdjust = 0; |
| 563 | if (NonLocalCost < Cost.NonLocalCost) |
| 564 | OtherNonLocalAdjust = Cost.NonLocalCost - NonLocalCost; |
| 565 | else |
| 566 | ThisNonLocalAdjust = NonLocalCost - Cost.NonLocalCost; |
| 567 | // Scale everything to make them comparable. |
| 568 | uint64_t ThisScaledCost = ThisLocalAdjust * LocalFreq; |
| 569 | // Check for overflow on that operation. |
| 570 | bool ThisOverflows = ThisLocalAdjust && (ThisScaledCost < ThisLocalAdjust || |
| 571 | ThisScaledCost < LocalFreq); |
| 572 | uint64_t OtherScaledCost = OtherLocalAdjust * Cost.LocalFreq; |
| 573 | // Check for overflow on the last operation. |
| 574 | bool OtherOverflows = |
| 575 | OtherLocalAdjust && |
| 576 | (OtherScaledCost < OtherLocalAdjust || OtherScaledCost < Cost.LocalFreq); |
| 577 | // Add the non-local costs. |
| 578 | ThisOverflows |= ThisNonLocalAdjust && |
| 579 | ThisScaledCost + ThisNonLocalAdjust < ThisNonLocalAdjust; |
| 580 | ThisScaledCost += ThisNonLocalAdjust; |
| 581 | OtherOverflows |= OtherNonLocalAdjust && |
| 582 | OtherScaledCost + OtherNonLocalAdjust < OtherNonLocalAdjust; |
| 583 | OtherScaledCost += OtherNonLocalAdjust; |
| 584 | // If both overflows, we cannot compare without additional |
| 585 | // precision, e.g., APInt. Just give up on that case. |
| 586 | if (ThisOverflows && OtherOverflows) |
| 587 | return false; |
| 588 | // If one overflows but not the other, we can still compare. |
| 589 | if (ThisOverflows || OtherOverflows) |
| 590 | return ThisOverflows < OtherOverflows; |
| 591 | // Otherwise, just compare the values. |
| 592 | return ThisScaledCost < OtherScaledCost; |
| 593 | } |
| 594 | |
| 595 | bool RegBankSelect::MappingCost::operator==(const MappingCost &Cost) const { |
| 596 | return LocalCost == Cost.LocalCost && NonLocalCost == Cost.NonLocalCost && |
| 597 | LocalFreq == Cost.LocalFreq; |
| 598 | } |