Chris Lattner | 0d5644b | 2003-01-13 00:26:36 +0000 | [diff] [blame] | 1 | //===-- TargetInstrInfo.cpp - Target Instruction Information --------------===// |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 2 | // |
John Criswell | 482202a | 2003-10-20 19:43:21 +0000 | [diff] [blame] | 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
Chris Lattner | f3ebc3f | 2007-12-29 20:36:04 +0000 | [diff] [blame] | 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 7 | // |
John Criswell | 482202a | 2003-10-20 19:43:21 +0000 | [diff] [blame] | 8 | //===----------------------------------------------------------------------===// |
Chris Lattner | 910b82f | 2002-10-28 23:55:33 +0000 | [diff] [blame] | 9 | // |
Chris Lattner | f6932b7 | 2005-01-19 06:53:34 +0000 | [diff] [blame] | 10 | // This file implements the TargetInstrInfo class. |
Chris Lattner | 910b82f | 2002-10-28 23:55:33 +0000 | [diff] [blame] | 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
Eric Christopher | 4fdc765 | 2014-06-11 16:59:33 +0000 | [diff] [blame] | 14 | #include "llvm/Target/TargetInstrInfo.h" |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 15 | #include "llvm/CodeGen/MachineFrameInfo.h" |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 16 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 17 | #include "llvm/CodeGen/MachineMemOperand.h" |
| 18 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 19 | #include "llvm/CodeGen/PseudoSourceValue.h" |
| 20 | #include "llvm/CodeGen/ScoreboardHazardRecognizer.h" |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 21 | #include "llvm/CodeGen/StackMaps.h" |
Matthias Braun | 88e2131 | 2015-06-13 03:42:11 +0000 | [diff] [blame] | 22 | #include "llvm/CodeGen/TargetSchedule.h" |
Andrew Trick | 10d5be4 | 2013-11-17 01:36:23 +0000 | [diff] [blame] | 23 | #include "llvm/IR/DataLayout.h" |
Evan Cheng | 49d4c0b | 2010-10-06 06:27:31 +0000 | [diff] [blame] | 24 | #include "llvm/MC/MCAsmInfo.h" |
Evan Cheng | 8264e27 | 2011-06-29 01:14:12 +0000 | [diff] [blame] | 25 | #include "llvm/MC/MCInstrItineraries.h" |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 26 | #include "llvm/Support/CommandLine.h" |
Chris Lattner | 0161419 | 2009-08-02 04:58:19 +0000 | [diff] [blame] | 27 | #include "llvm/Support/ErrorHandling.h" |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 28 | #include "llvm/Support/raw_ostream.h" |
Michael Kuperstein | 698ea3b | 2015-01-08 11:59:43 +0000 | [diff] [blame] | 29 | #include "llvm/Target/TargetFrameLowering.h" |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 30 | #include "llvm/Target/TargetLowering.h" |
| 31 | #include "llvm/Target/TargetMachine.h" |
Chandler Carruth | ed0881b | 2012-12-03 16:50:05 +0000 | [diff] [blame] | 32 | #include "llvm/Target/TargetRegisterInfo.h" |
Nick Lewycky | 0de20af | 2010-12-19 20:43:38 +0000 | [diff] [blame] | 33 | #include <cctype> |
Eugene Zelenko | ecefe5a | 2016-02-02 18:20:45 +0000 | [diff] [blame] | 34 | |
Chris Lattner | f6932b7 | 2005-01-19 06:53:34 +0000 | [diff] [blame] | 35 | using namespace llvm; |
Chris Lattner | 910b82f | 2002-10-28 23:55:33 +0000 | [diff] [blame] | 36 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 37 | static cl::opt<bool> DisableHazardRecognizer( |
| 38 | "disable-sched-hazard", cl::Hidden, cl::init(false), |
| 39 | cl::desc("Disable hazard detection during preRA scheduling")); |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 40 | |
Chris Lattner | 0d5644b | 2003-01-13 00:26:36 +0000 | [diff] [blame] | 41 | TargetInstrInfo::~TargetInstrInfo() { |
Chris Lattner | 910b82f | 2002-10-28 23:55:33 +0000 | [diff] [blame] | 42 | } |
| 43 | |
Evan Cheng | 8d71a75 | 2011-06-27 21:26:13 +0000 | [diff] [blame] | 44 | const TargetRegisterClass* |
Evan Cheng | 6cc775f | 2011-06-28 19:10:37 +0000 | [diff] [blame] | 45 | TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum, |
Jakob Stoklund Olesen | 3c52f02 | 2012-05-07 22:10:26 +0000 | [diff] [blame] | 46 | const TargetRegisterInfo *TRI, |
| 47 | const MachineFunction &MF) const { |
Evan Cheng | 6cc775f | 2011-06-28 19:10:37 +0000 | [diff] [blame] | 48 | if (OpNum >= MCID.getNumOperands()) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 49 | return nullptr; |
Evan Cheng | 8d71a75 | 2011-06-27 21:26:13 +0000 | [diff] [blame] | 50 | |
Evan Cheng | 6cc775f | 2011-06-28 19:10:37 +0000 | [diff] [blame] | 51 | short RegClass = MCID.OpInfo[OpNum].RegClass; |
| 52 | if (MCID.OpInfo[OpNum].isLookupPtrRegClass()) |
Jakob Stoklund Olesen | 3c52f02 | 2012-05-07 22:10:26 +0000 | [diff] [blame] | 53 | return TRI->getPointerRegClass(MF, RegClass); |
Evan Cheng | 8d71a75 | 2011-06-27 21:26:13 +0000 | [diff] [blame] | 54 | |
| 55 | // Instructions like INSERT_SUBREG do not have fixed register classes. |
| 56 | if (RegClass < 0) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 57 | return nullptr; |
Evan Cheng | 8d71a75 | 2011-06-27 21:26:13 +0000 | [diff] [blame] | 58 | |
| 59 | // Otherwise just look it up normally. |
| 60 | return TRI->getRegClass(RegClass); |
| 61 | } |
| 62 | |
Chris Lattner | 0161419 | 2009-08-02 04:58:19 +0000 | [diff] [blame] | 63 | /// insertNoop - Insert a noop into the instruction stream at the specified |
| 64 | /// point. |
Andrew Trick | c416ba6 | 2010-12-24 04:28:06 +0000 | [diff] [blame] | 65 | void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB, |
Chris Lattner | 0161419 | 2009-08-02 04:58:19 +0000 | [diff] [blame] | 66 | MachineBasicBlock::iterator MI) const { |
| 67 | llvm_unreachable("Target didn't implement insertNoop!"); |
| 68 | } |
| 69 | |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 70 | /// Measure the specified inline asm to determine an approximation of its |
| 71 | /// length. |
Jim Grosbach | a3df87f | 2011-03-24 18:46:34 +0000 | [diff] [blame] | 72 | /// Comments (which run till the next SeparatorString or newline) do not |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 73 | /// count as an instruction. |
| 74 | /// Any other non-whitespace text is considered an instruction, with |
Jim Grosbach | a3df87f | 2011-03-24 18:46:34 +0000 | [diff] [blame] | 75 | /// multiple instructions separated by SeparatorString or newlines. |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 76 | /// Variable-length instructions are not handled here; this function |
| 77 | /// may be overloaded in the target code to do that. |
| 78 | unsigned TargetInstrInfo::getInlineAsmLength(const char *Str, |
Chris Lattner | e9a75a6 | 2009-08-22 21:43:10 +0000 | [diff] [blame] | 79 | const MCAsmInfo &MAI) const { |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 80 | // Count the number of instructions in the asm. |
| 81 | bool atInsnStart = true; |
| 82 | unsigned Length = 0; |
| 83 | for (; *Str; ++Str) { |
Jim Grosbach | a3df87f | 2011-03-24 18:46:34 +0000 | [diff] [blame] | 84 | if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(), |
| 85 | strlen(MAI.getSeparatorString())) == 0) |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 86 | atInsnStart = true; |
Guy Benyei | 83c74e9 | 2013-02-12 21:21:59 +0000 | [diff] [blame] | 87 | if (atInsnStart && !std::isspace(static_cast<unsigned char>(*Str))) { |
Chris Lattner | e9a75a6 | 2009-08-22 21:43:10 +0000 | [diff] [blame] | 88 | Length += MAI.getMaxInstLength(); |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 89 | atInsnStart = false; |
| 90 | } |
Chris Lattner | e9a75a6 | 2009-08-22 21:43:10 +0000 | [diff] [blame] | 91 | if (atInsnStart && strncmp(Str, MAI.getCommentString(), |
| 92 | strlen(MAI.getCommentString())) == 0) |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 93 | atInsnStart = false; |
| 94 | } |
Andrew Trick | c416ba6 | 2010-12-24 04:28:06 +0000 | [diff] [blame] | 95 | |
Chris Lattner | e98a3c3 | 2009-08-02 05:20:37 +0000 | [diff] [blame] | 96 | return Length; |
| 97 | } |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 98 | |
| 99 | /// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything |
| 100 | /// after it, replacing it with an unconditional branch to NewDest. |
| 101 | void |
| 102 | TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, |
| 103 | MachineBasicBlock *NewDest) const { |
| 104 | MachineBasicBlock *MBB = Tail->getParent(); |
| 105 | |
| 106 | // Remove all the old successors of MBB from the CFG. |
| 107 | while (!MBB->succ_empty()) |
| 108 | MBB->removeSuccessor(MBB->succ_begin()); |
| 109 | |
| 110 | // Remove all the dead instructions from the end of MBB. |
| 111 | MBB->erase(Tail, MBB->end()); |
| 112 | |
| 113 | // If MBB isn't immediately before MBB, insert a branch to it. |
| 114 | if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest)) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 115 | InsertBranch(*MBB, NewDest, nullptr, SmallVector<MachineOperand, 0>(), |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 116 | Tail->getDebugLoc()); |
| 117 | MBB->addSuccessor(NewDest); |
| 118 | } |
| 119 | |
Andrew Kaylor | 16c4da0 | 2015-09-28 20:33:22 +0000 | [diff] [blame] | 120 | MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr *MI, |
| 121 | bool NewMI, |
| 122 | unsigned Idx1, |
| 123 | unsigned Idx2) const { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 124 | const MCInstrDesc &MCID = MI->getDesc(); |
| 125 | bool HasDef = MCID.getNumDefs(); |
| 126 | if (HasDef && !MI->getOperand(0).isReg()) |
| 127 | // No idea how to commute this instruction. Target should implement its own. |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 128 | return nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 129 | |
Richard Trieu | e778e87 | 2015-09-28 22:54:43 +0000 | [diff] [blame] | 130 | unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1; |
| 131 | unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2; |
Andrew Kaylor | 16c4da0 | 2015-09-28 20:33:22 +0000 | [diff] [blame] | 132 | assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) && |
| 133 | CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 && |
| 134 | "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands."); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 135 | assert(MI->getOperand(Idx1).isReg() && MI->getOperand(Idx2).isReg() && |
| 136 | "This only knows how to commute register operands so far"); |
Andrew Kaylor | 16c4da0 | 2015-09-28 20:33:22 +0000 | [diff] [blame] | 137 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 138 | unsigned Reg0 = HasDef ? MI->getOperand(0).getReg() : 0; |
| 139 | unsigned Reg1 = MI->getOperand(Idx1).getReg(); |
| 140 | unsigned Reg2 = MI->getOperand(Idx2).getReg(); |
| 141 | unsigned SubReg0 = HasDef ? MI->getOperand(0).getSubReg() : 0; |
| 142 | unsigned SubReg1 = MI->getOperand(Idx1).getSubReg(); |
| 143 | unsigned SubReg2 = MI->getOperand(Idx2).getSubReg(); |
| 144 | bool Reg1IsKill = MI->getOperand(Idx1).isKill(); |
| 145 | bool Reg2IsKill = MI->getOperand(Idx2).isKill(); |
Andrea Di Biagio | c84b5bd | 2015-04-30 21:03:29 +0000 | [diff] [blame] | 146 | bool Reg1IsUndef = MI->getOperand(Idx1).isUndef(); |
| 147 | bool Reg2IsUndef = MI->getOperand(Idx2).isUndef(); |
Pete Cooper | 451755d | 2015-04-30 23:14:14 +0000 | [diff] [blame] | 148 | bool Reg1IsInternal = MI->getOperand(Idx1).isInternalRead(); |
| 149 | bool Reg2IsInternal = MI->getOperand(Idx2).isInternalRead(); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 150 | // If destination is tied to either of the commuted source register, then |
| 151 | // it must be updated. |
| 152 | if (HasDef && Reg0 == Reg1 && |
| 153 | MI->getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) { |
| 154 | Reg2IsKill = false; |
| 155 | Reg0 = Reg2; |
| 156 | SubReg0 = SubReg2; |
| 157 | } else if (HasDef && Reg0 == Reg2 && |
| 158 | MI->getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) { |
| 159 | Reg1IsKill = false; |
| 160 | Reg0 = Reg1; |
| 161 | SubReg0 = SubReg1; |
| 162 | } |
| 163 | |
| 164 | if (NewMI) { |
| 165 | // Create a new instruction. |
| 166 | MachineFunction &MF = *MI->getParent()->getParent(); |
| 167 | MI = MF.CloneMachineInstr(MI); |
| 168 | } |
| 169 | |
| 170 | if (HasDef) { |
| 171 | MI->getOperand(0).setReg(Reg0); |
| 172 | MI->getOperand(0).setSubReg(SubReg0); |
| 173 | } |
| 174 | MI->getOperand(Idx2).setReg(Reg1); |
| 175 | MI->getOperand(Idx1).setReg(Reg2); |
| 176 | MI->getOperand(Idx2).setSubReg(SubReg1); |
| 177 | MI->getOperand(Idx1).setSubReg(SubReg2); |
| 178 | MI->getOperand(Idx2).setIsKill(Reg1IsKill); |
| 179 | MI->getOperand(Idx1).setIsKill(Reg2IsKill); |
Andrea Di Biagio | c84b5bd | 2015-04-30 21:03:29 +0000 | [diff] [blame] | 180 | MI->getOperand(Idx2).setIsUndef(Reg1IsUndef); |
| 181 | MI->getOperand(Idx1).setIsUndef(Reg2IsUndef); |
Pete Cooper | 451755d | 2015-04-30 23:14:14 +0000 | [diff] [blame] | 182 | MI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal); |
| 183 | MI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 184 | return MI; |
| 185 | } |
| 186 | |
Andrew Kaylor | 16c4da0 | 2015-09-28 20:33:22 +0000 | [diff] [blame] | 187 | MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr *MI, |
| 188 | bool NewMI, |
| 189 | unsigned OpIdx1, |
| 190 | unsigned OpIdx2) const { |
| 191 | // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose |
| 192 | // any commutable operand, which is done in findCommutedOpIndices() method |
| 193 | // called below. |
| 194 | if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) && |
| 195 | !findCommutedOpIndices(MI, OpIdx1, OpIdx2)) { |
| 196 | assert(MI->isCommutable() && |
| 197 | "Precondition violation: MI must be commutable."); |
| 198 | return nullptr; |
| 199 | } |
| 200 | return commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); |
| 201 | } |
| 202 | |
| 203 | bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1, |
| 204 | unsigned &ResultIdx2, |
| 205 | unsigned CommutableOpIdx1, |
| 206 | unsigned CommutableOpIdx2) { |
| 207 | if (ResultIdx1 == CommuteAnyOperandIndex && |
| 208 | ResultIdx2 == CommuteAnyOperandIndex) { |
| 209 | ResultIdx1 = CommutableOpIdx1; |
| 210 | ResultIdx2 = CommutableOpIdx2; |
| 211 | } else if (ResultIdx1 == CommuteAnyOperandIndex) { |
| 212 | if (ResultIdx2 == CommutableOpIdx1) |
| 213 | ResultIdx1 = CommutableOpIdx2; |
| 214 | else if (ResultIdx2 == CommutableOpIdx2) |
| 215 | ResultIdx1 = CommutableOpIdx1; |
| 216 | else |
| 217 | return false; |
| 218 | } else if (ResultIdx2 == CommuteAnyOperandIndex) { |
| 219 | if (ResultIdx1 == CommutableOpIdx1) |
| 220 | ResultIdx2 = CommutableOpIdx2; |
| 221 | else if (ResultIdx1 == CommutableOpIdx2) |
| 222 | ResultIdx2 = CommutableOpIdx1; |
| 223 | else |
| 224 | return false; |
| 225 | } else |
| 226 | // Check that the result operand indices match the given commutable |
| 227 | // operand indices. |
| 228 | return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) || |
| 229 | (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1); |
| 230 | |
| 231 | return true; |
| 232 | } |
| 233 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 234 | bool TargetInstrInfo::findCommutedOpIndices(MachineInstr *MI, |
| 235 | unsigned &SrcOpIdx1, |
| 236 | unsigned &SrcOpIdx2) const { |
| 237 | assert(!MI->isBundle() && |
| 238 | "TargetInstrInfo::findCommutedOpIndices() can't handle bundles"); |
| 239 | |
| 240 | const MCInstrDesc &MCID = MI->getDesc(); |
| 241 | if (!MCID.isCommutable()) |
| 242 | return false; |
Andrew Kaylor | 16c4da0 | 2015-09-28 20:33:22 +0000 | [diff] [blame] | 243 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 244 | // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this |
| 245 | // is not true, then the target must implement this. |
Andrew Kaylor | 16c4da0 | 2015-09-28 20:33:22 +0000 | [diff] [blame] | 246 | unsigned CommutableOpIdx1 = MCID.getNumDefs(); |
| 247 | unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1; |
| 248 | if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, |
| 249 | CommutableOpIdx1, CommutableOpIdx2)) |
| 250 | return false; |
| 251 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 252 | if (!MI->getOperand(SrcOpIdx1).isReg() || |
| 253 | !MI->getOperand(SrcOpIdx2).isReg()) |
| 254 | // No idea. |
| 255 | return false; |
| 256 | return true; |
| 257 | } |
| 258 | |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 259 | bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const { |
| 260 | if (!MI.isTerminator()) return false; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 261 | |
| 262 | // Conditional branch is a special case. |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 263 | if (MI.isBranch() && !MI.isBarrier()) |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 264 | return true; |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 265 | if (!MI.isPredicable()) |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 266 | return true; |
| 267 | return !isPredicated(MI); |
| 268 | } |
| 269 | |
Ahmed Bougacha | c88bf54 | 2015-06-11 19:30:37 +0000 | [diff] [blame] | 270 | bool TargetInstrInfo::PredicateInstruction( |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 271 | MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 272 | bool MadeChange = false; |
| 273 | |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 274 | assert(!MI.isBundle() && |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 275 | "TargetInstrInfo::PredicateInstruction() can't handle bundles"); |
| 276 | |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 277 | const MCInstrDesc &MCID = MI.getDesc(); |
| 278 | if (!MI.isPredicable()) |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 279 | return false; |
| 280 | |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 281 | for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 282 | if (MCID.OpInfo[i].isPredicate()) { |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 283 | MachineOperand &MO = MI.getOperand(i); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 284 | if (MO.isReg()) { |
| 285 | MO.setReg(Pred[j].getReg()); |
| 286 | MadeChange = true; |
| 287 | } else if (MO.isImm()) { |
| 288 | MO.setImm(Pred[j].getImm()); |
| 289 | MadeChange = true; |
| 290 | } else if (MO.isMBB()) { |
| 291 | MO.setMBB(Pred[j].getMBB()); |
| 292 | MadeChange = true; |
| 293 | } |
| 294 | ++j; |
| 295 | } |
| 296 | } |
| 297 | return MadeChange; |
| 298 | } |
| 299 | |
| 300 | bool TargetInstrInfo::hasLoadFromStackSlot(const MachineInstr *MI, |
| 301 | const MachineMemOperand *&MMO, |
| 302 | int &FrameIndex) const { |
| 303 | for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), |
| 304 | oe = MI->memoperands_end(); |
| 305 | o != oe; |
| 306 | ++o) { |
Nick Lewycky | aad475b | 2014-04-15 07:22:52 +0000 | [diff] [blame] | 307 | if ((*o)->isLoad()) { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 308 | if (const FixedStackPseudoSourceValue *Value = |
Nick Lewycky | aad475b | 2014-04-15 07:22:52 +0000 | [diff] [blame] | 309 | dyn_cast_or_null<FixedStackPseudoSourceValue>( |
| 310 | (*o)->getPseudoValue())) { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 311 | FrameIndex = Value->getFrameIndex(); |
| 312 | MMO = *o; |
| 313 | return true; |
| 314 | } |
Nick Lewycky | aad475b | 2014-04-15 07:22:52 +0000 | [diff] [blame] | 315 | } |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 316 | } |
| 317 | return false; |
| 318 | } |
| 319 | |
| 320 | bool TargetInstrInfo::hasStoreToStackSlot(const MachineInstr *MI, |
| 321 | const MachineMemOperand *&MMO, |
| 322 | int &FrameIndex) const { |
| 323 | for (MachineInstr::mmo_iterator o = MI->memoperands_begin(), |
| 324 | oe = MI->memoperands_end(); |
| 325 | o != oe; |
| 326 | ++o) { |
Nick Lewycky | aad475b | 2014-04-15 07:22:52 +0000 | [diff] [blame] | 327 | if ((*o)->isStore()) { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 328 | if (const FixedStackPseudoSourceValue *Value = |
Nick Lewycky | aad475b | 2014-04-15 07:22:52 +0000 | [diff] [blame] | 329 | dyn_cast_or_null<FixedStackPseudoSourceValue>( |
| 330 | (*o)->getPseudoValue())) { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 331 | FrameIndex = Value->getFrameIndex(); |
| 332 | MMO = *o; |
| 333 | return true; |
| 334 | } |
Nick Lewycky | aad475b | 2014-04-15 07:22:52 +0000 | [diff] [blame] | 335 | } |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 336 | } |
| 337 | return false; |
| 338 | } |
| 339 | |
Andrew Trick | 10d5be4 | 2013-11-17 01:36:23 +0000 | [diff] [blame] | 340 | bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC, |
| 341 | unsigned SubIdx, unsigned &Size, |
| 342 | unsigned &Offset, |
Eric Christopher | 7585fb2 | 2015-03-19 23:06:21 +0000 | [diff] [blame] | 343 | const MachineFunction &MF) const { |
Andrew Trick | 10d5be4 | 2013-11-17 01:36:23 +0000 | [diff] [blame] | 344 | if (!SubIdx) { |
| 345 | Size = RC->getSize(); |
| 346 | Offset = 0; |
| 347 | return true; |
| 348 | } |
Eric Christopher | 7585fb2 | 2015-03-19 23:06:21 +0000 | [diff] [blame] | 349 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
| 350 | unsigned BitSize = TRI->getSubRegIdxSize(SubIdx); |
Andrew Trick | 10d5be4 | 2013-11-17 01:36:23 +0000 | [diff] [blame] | 351 | // Convert bit size to byte size to be consistent with |
| 352 | // MCRegisterClass::getSize(). |
| 353 | if (BitSize % 8) |
| 354 | return false; |
| 355 | |
Eric Christopher | 7585fb2 | 2015-03-19 23:06:21 +0000 | [diff] [blame] | 356 | int BitOffset = TRI->getSubRegIdxOffset(SubIdx); |
Andrew Trick | 10d5be4 | 2013-11-17 01:36:23 +0000 | [diff] [blame] | 357 | if (BitOffset < 0 || BitOffset % 8) |
| 358 | return false; |
| 359 | |
| 360 | Size = BitSize /= 8; |
| 361 | Offset = (unsigned)BitOffset / 8; |
| 362 | |
| 363 | assert(RC->getSize() >= (Offset + Size) && "bad subregister range"); |
| 364 | |
Mehdi Amini | bd7287e | 2015-07-16 06:11:10 +0000 | [diff] [blame] | 365 | if (!MF.getDataLayout().isLittleEndian()) { |
Andrew Trick | 10d5be4 | 2013-11-17 01:36:23 +0000 | [diff] [blame] | 366 | Offset = RC->getSize() - (Offset + Size); |
| 367 | } |
| 368 | return true; |
| 369 | } |
| 370 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 371 | void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB, |
| 372 | MachineBasicBlock::iterator I, |
| 373 | unsigned DestReg, |
| 374 | unsigned SubIdx, |
| 375 | const MachineInstr *Orig, |
| 376 | const TargetRegisterInfo &TRI) const { |
| 377 | MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig); |
| 378 | MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI); |
| 379 | MBB.insert(I, MI); |
| 380 | } |
| 381 | |
| 382 | bool |
| 383 | TargetInstrInfo::produceSameValue(const MachineInstr *MI0, |
| 384 | const MachineInstr *MI1, |
| 385 | const MachineRegisterInfo *MRI) const { |
| 386 | return MI0->isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs); |
| 387 | } |
| 388 | |
| 389 | MachineInstr *TargetInstrInfo::duplicate(MachineInstr *Orig, |
| 390 | MachineFunction &MF) const { |
| 391 | assert(!Orig->isNotDuplicable() && |
| 392 | "Instruction cannot be duplicated"); |
| 393 | return MF.CloneMachineInstr(Orig); |
| 394 | } |
| 395 | |
| 396 | // If the COPY instruction in MI can be folded to a stack operation, return |
| 397 | // the register class to use. |
| 398 | static const TargetRegisterClass *canFoldCopy(const MachineInstr *MI, |
| 399 | unsigned FoldIdx) { |
| 400 | assert(MI->isCopy() && "MI must be a COPY instruction"); |
| 401 | if (MI->getNumOperands() != 2) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 402 | return nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 403 | assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand"); |
| 404 | |
| 405 | const MachineOperand &FoldOp = MI->getOperand(FoldIdx); |
| 406 | const MachineOperand &LiveOp = MI->getOperand(1-FoldIdx); |
| 407 | |
| 408 | if (FoldOp.getSubReg() || LiveOp.getSubReg()) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 409 | return nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 410 | |
| 411 | unsigned FoldReg = FoldOp.getReg(); |
| 412 | unsigned LiveReg = LiveOp.getReg(); |
| 413 | |
| 414 | assert(TargetRegisterInfo::isVirtualRegister(FoldReg) && |
| 415 | "Cannot fold physregs"); |
| 416 | |
| 417 | const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); |
| 418 | const TargetRegisterClass *RC = MRI.getRegClass(FoldReg); |
| 419 | |
| 420 | if (TargetRegisterInfo::isPhysicalRegister(LiveOp.getReg())) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 421 | return RC->contains(LiveOp.getReg()) ? RC : nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 422 | |
| 423 | if (RC->hasSubClassEq(MRI.getRegClass(LiveReg))) |
| 424 | return RC; |
| 425 | |
| 426 | // FIXME: Allow folding when register classes are memory compatible. |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 427 | return nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 428 | } |
| 429 | |
Rafael Espindola | 6865d6f | 2014-09-15 18:32:58 +0000 | [diff] [blame] | 430 | void TargetInstrInfo::getNoopForMachoTarget(MCInst &NopInst) const { |
| 431 | llvm_unreachable("Not a MachO target"); |
| 432 | } |
| 433 | |
Benjamin Kramer | f1362f6 | 2015-02-28 12:04:00 +0000 | [diff] [blame] | 434 | static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr *MI, |
| 435 | ArrayRef<unsigned> Ops, int FrameIndex, |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 436 | const TargetInstrInfo &TII) { |
| 437 | unsigned StartIdx = 0; |
| 438 | switch (MI->getOpcode()) { |
| 439 | case TargetOpcode::STACKMAP: |
| 440 | StartIdx = 2; // Skip ID, nShadowBytes. |
| 441 | break; |
| 442 | case TargetOpcode::PATCHPOINT: { |
| 443 | // For PatchPoint, the call args are not foldable. |
| 444 | PatchPointOpers opers(MI); |
| 445 | StartIdx = opers.getVarIdx(); |
| 446 | break; |
| 447 | } |
| 448 | default: |
| 449 | llvm_unreachable("unexpected stackmap opcode"); |
| 450 | } |
| 451 | |
| 452 | // Return false if any operands requested for folding are not foldable (not |
| 453 | // part of the stackmap's live values). |
Benjamin Kramer | f1362f6 | 2015-02-28 12:04:00 +0000 | [diff] [blame] | 454 | for (unsigned Op : Ops) { |
| 455 | if (Op < StartIdx) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 456 | return nullptr; |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 457 | } |
| 458 | |
| 459 | MachineInstr *NewMI = |
| 460 | MF.CreateMachineInstr(TII.get(MI->getOpcode()), MI->getDebugLoc(), true); |
| 461 | MachineInstrBuilder MIB(MF, NewMI); |
| 462 | |
| 463 | // No need to fold return, the meta data, and function arguments |
| 464 | for (unsigned i = 0; i < StartIdx; ++i) |
| 465 | MIB.addOperand(MI->getOperand(i)); |
| 466 | |
| 467 | for (unsigned i = StartIdx; i < MI->getNumOperands(); ++i) { |
| 468 | MachineOperand &MO = MI->getOperand(i); |
| 469 | if (std::find(Ops.begin(), Ops.end(), i) != Ops.end()) { |
| 470 | unsigned SpillSize; |
| 471 | unsigned SpillOffset; |
| 472 | // Compute the spill slot size and offset. |
| 473 | const TargetRegisterClass *RC = |
| 474 | MF.getRegInfo().getRegClass(MO.getReg()); |
Eric Christopher | 7585fb2 | 2015-03-19 23:06:21 +0000 | [diff] [blame] | 475 | bool Valid = |
| 476 | TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF); |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 477 | if (!Valid) |
| 478 | report_fatal_error("cannot spill patchpoint subregister operand"); |
| 479 | MIB.addImm(StackMaps::IndirectMemRefOp); |
| 480 | MIB.addImm(SpillSize); |
| 481 | MIB.addFrameIndex(FrameIndex); |
Lang Hames | 2ce64a7 | 2013-12-07 03:30:59 +0000 | [diff] [blame] | 482 | MIB.addImm(SpillOffset); |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 483 | } |
| 484 | else |
| 485 | MIB.addOperand(MO); |
| 486 | } |
| 487 | return NewMI; |
| 488 | } |
| 489 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 490 | /// foldMemoryOperand - Attempt to fold a load or store of the specified stack |
| 491 | /// slot into the specified machine instruction for the specified operand(s). |
| 492 | /// If this is possible, a new instruction is returned with the specified |
| 493 | /// operand folded, otherwise NULL is returned. The client is responsible for |
| 494 | /// removing the old instruction and adding the new one in the instruction |
| 495 | /// stream. |
Benjamin Kramer | f1362f6 | 2015-02-28 12:04:00 +0000 | [diff] [blame] | 496 | MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, |
| 497 | ArrayRef<unsigned> Ops, |
| 498 | int FI) const { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 499 | unsigned Flags = 0; |
| 500 | for (unsigned i = 0, e = Ops.size(); i != e; ++i) |
| 501 | if (MI->getOperand(Ops[i]).isDef()) |
| 502 | Flags |= MachineMemOperand::MOStore; |
| 503 | else |
| 504 | Flags |= MachineMemOperand::MOLoad; |
| 505 | |
| 506 | MachineBasicBlock *MBB = MI->getParent(); |
| 507 | assert(MBB && "foldMemoryOperand needs an inserted instruction"); |
| 508 | MachineFunction &MF = *MBB->getParent(); |
| 509 | |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 510 | MachineInstr *NewMI = nullptr; |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 511 | |
| 512 | if (MI->getOpcode() == TargetOpcode::STACKMAP || |
| 513 | MI->getOpcode() == TargetOpcode::PATCHPOINT) { |
| 514 | // Fold stackmap/patchpoint. |
| 515 | NewMI = foldPatchpoint(MF, MI, Ops, FI, *this); |
Keno Fischer | e70b31f | 2015-06-08 20:09:58 +0000 | [diff] [blame] | 516 | if (NewMI) |
| 517 | MBB->insert(MI, NewMI); |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 518 | } else { |
| 519 | // Ask the target to do the actual folding. |
Keno Fischer | e70b31f | 2015-06-08 20:09:58 +0000 | [diff] [blame] | 520 | NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI); |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 521 | } |
Keno Fischer | e70b31f | 2015-06-08 20:09:58 +0000 | [diff] [blame] | 522 | |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 523 | if (NewMI) { |
Andrew Trick | a9f4d92 | 2013-11-14 23:45:04 +0000 | [diff] [blame] | 524 | NewMI->setMemRefs(MI->memoperands_begin(), MI->memoperands_end()); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 525 | // Add a memory operand, foldMemoryOperandImpl doesn't do that. |
| 526 | assert((!(Flags & MachineMemOperand::MOStore) || |
| 527 | NewMI->mayStore()) && |
| 528 | "Folded a def to a non-store!"); |
| 529 | assert((!(Flags & MachineMemOperand::MOLoad) || |
| 530 | NewMI->mayLoad()) && |
| 531 | "Folded a use to a non-load!"); |
| 532 | const MachineFrameInfo &MFI = *MF.getFrameInfo(); |
| 533 | assert(MFI.getObjectOffset(FI) != -1); |
Alex Lorenz | e40c8a2 | 2015-08-11 23:09:45 +0000 | [diff] [blame] | 534 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
| 535 | MachinePointerInfo::getFixedStack(MF, FI), Flags, MFI.getObjectSize(FI), |
| 536 | MFI.getObjectAlignment(FI)); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 537 | NewMI->addMemOperand(MF, MMO); |
| 538 | |
Keno Fischer | e70b31f | 2015-06-08 20:09:58 +0000 | [diff] [blame] | 539 | return NewMI; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 540 | } |
| 541 | |
| 542 | // Straight COPY may fold as load/store. |
| 543 | if (!MI->isCopy() || Ops.size() != 1) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 544 | return nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 545 | |
| 546 | const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]); |
| 547 | if (!RC) |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 548 | return nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 549 | |
| 550 | const MachineOperand &MO = MI->getOperand(1-Ops[0]); |
| 551 | MachineBasicBlock::iterator Pos = MI; |
Eric Christopher | fc6de42 | 2014-08-05 02:39:49 +0000 | [diff] [blame] | 552 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 553 | |
| 554 | if (Flags == MachineMemOperand::MOStore) |
| 555 | storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI); |
| 556 | else |
| 557 | loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI); |
| 558 | return --Pos; |
| 559 | } |
| 560 | |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 561 | bool TargetInstrInfo::hasReassociableOperands( |
| 562 | const MachineInstr &Inst, const MachineBasicBlock *MBB) const { |
| 563 | const MachineOperand &Op1 = Inst.getOperand(1); |
| 564 | const MachineOperand &Op2 = Inst.getOperand(2); |
| 565 | const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); |
| 566 | |
| 567 | // We need virtual register definitions for the operands that we will |
| 568 | // reassociate. |
| 569 | MachineInstr *MI1 = nullptr; |
| 570 | MachineInstr *MI2 = nullptr; |
| 571 | if (Op1.isReg() && TargetRegisterInfo::isVirtualRegister(Op1.getReg())) |
| 572 | MI1 = MRI.getUniqueVRegDef(Op1.getReg()); |
| 573 | if (Op2.isReg() && TargetRegisterInfo::isVirtualRegister(Op2.getReg())) |
| 574 | MI2 = MRI.getUniqueVRegDef(Op2.getReg()); |
| 575 | |
| 576 | // And they need to be in the trace (otherwise, they won't have a depth). |
Rafael Espindola | 84921b9 | 2015-10-24 23:11:13 +0000 | [diff] [blame] | 577 | return MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB; |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 578 | } |
| 579 | |
| 580 | bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst, |
| 581 | bool &Commuted) const { |
| 582 | const MachineBasicBlock *MBB = Inst.getParent(); |
| 583 | const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); |
| 584 | MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg()); |
| 585 | MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg()); |
| 586 | unsigned AssocOpcode = Inst.getOpcode(); |
| 587 | |
| 588 | // If only one operand has the same opcode and it's the second source operand, |
| 589 | // the operands must be commuted. |
| 590 | Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode; |
| 591 | if (Commuted) |
| 592 | std::swap(MI1, MI2); |
| 593 | |
| 594 | // 1. The previous instruction must be the same type as Inst. |
| 595 | // 2. The previous instruction must have virtual register definitions for its |
| 596 | // operands in the same basic block as Inst. |
| 597 | // 3. The previous instruction's result must only be used by Inst. |
Rafael Espindola | 84921b9 | 2015-10-24 23:11:13 +0000 | [diff] [blame] | 598 | return MI1->getOpcode() == AssocOpcode && |
| 599 | hasReassociableOperands(*MI1, MBB) && |
| 600 | MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()); |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 601 | } |
| 602 | |
| 603 | // 1. The operation must be associative and commutative. |
| 604 | // 2. The instruction must have virtual register definitions for its |
| 605 | // operands in the same basic block. |
| 606 | // 3. The instruction must have a reassociable sibling. |
| 607 | bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst, |
| 608 | bool &Commuted) const { |
Rafael Espindola | 84921b9 | 2015-10-24 23:11:13 +0000 | [diff] [blame] | 609 | return isAssociativeAndCommutative(Inst) && |
| 610 | hasReassociableOperands(Inst, Inst.getParent()) && |
| 611 | hasReassociableSibling(Inst, Commuted); |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 612 | } |
| 613 | |
| 614 | // The concept of the reassociation pass is that these operations can benefit |
| 615 | // from this kind of transformation: |
| 616 | // |
| 617 | // A = ? op ? |
| 618 | // B = A op X (Prev) |
| 619 | // C = B op Y (Root) |
| 620 | // --> |
| 621 | // A = ? op ? |
| 622 | // B = X op Y |
| 623 | // C = A op B |
| 624 | // |
| 625 | // breaking the dependency between A and B, allowing them to be executed in |
| 626 | // parallel (or back-to-back in a pipeline) instead of depending on each other. |
| 627 | |
| 628 | // FIXME: This has the potential to be expensive (compile time) while not |
| 629 | // improving the code at all. Some ways to limit the overhead: |
| 630 | // 1. Track successful transforms; bail out if hit rate gets too low. |
| 631 | // 2. Only enable at -O3 or some other non-default optimization level. |
| 632 | // 3. Pre-screen pattern candidates here: if an operand of the previous |
| 633 | // instruction is known to not increase the critical path, then don't match |
| 634 | // that pattern. |
| 635 | bool TargetInstrInfo::getMachineCombinerPatterns( |
| 636 | MachineInstr &Root, |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 637 | SmallVectorImpl<MachineCombinerPattern> &Patterns) const { |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 638 | bool Commute; |
| 639 | if (isReassociationCandidate(Root, Commute)) { |
| 640 | // We found a sequence of instructions that may be suitable for a |
| 641 | // reassociation of operands to increase ILP. Specify each commutation |
| 642 | // possibility for the Prev instruction in the sequence and let the |
| 643 | // machine combiner decide if changing the operands is worthwhile. |
| 644 | if (Commute) { |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 645 | Patterns.push_back(MachineCombinerPattern::REASSOC_AX_YB); |
| 646 | Patterns.push_back(MachineCombinerPattern::REASSOC_XA_YB); |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 647 | } else { |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 648 | Patterns.push_back(MachineCombinerPattern::REASSOC_AX_BY); |
| 649 | Patterns.push_back(MachineCombinerPattern::REASSOC_XA_BY); |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 650 | } |
| 651 | return true; |
| 652 | } |
| 653 | |
| 654 | return false; |
| 655 | } |
| 656 | |
| 657 | /// Attempt the reassociation transformation to reduce critical path length. |
| 658 | /// See the above comments before getMachineCombinerPatterns(). |
| 659 | void TargetInstrInfo::reassociateOps( |
| 660 | MachineInstr &Root, MachineInstr &Prev, |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 661 | MachineCombinerPattern Pattern, |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 662 | SmallVectorImpl<MachineInstr *> &InsInstrs, |
| 663 | SmallVectorImpl<MachineInstr *> &DelInstrs, |
| 664 | DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { |
| 665 | MachineFunction *MF = Root.getParent()->getParent(); |
| 666 | MachineRegisterInfo &MRI = MF->getRegInfo(); |
| 667 | const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); |
| 668 | const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); |
| 669 | const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI); |
| 670 | |
| 671 | // This array encodes the operand index for each parameter because the |
| 672 | // operands may be commuted. Each row corresponds to a pattern value, |
| 673 | // and each column specifies the index of A, B, X, Y. |
| 674 | unsigned OpIdx[4][4] = { |
| 675 | { 1, 1, 2, 2 }, |
| 676 | { 1, 2, 2, 1 }, |
| 677 | { 2, 1, 1, 2 }, |
| 678 | { 2, 2, 1, 1 } |
| 679 | }; |
| 680 | |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 681 | int Row; |
| 682 | switch (Pattern) { |
| 683 | case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break; |
| 684 | case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break; |
| 685 | case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break; |
| 686 | case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break; |
| 687 | default: llvm_unreachable("unexpected MachineCombinerPattern"); |
| 688 | } |
| 689 | |
| 690 | MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]); |
| 691 | MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]); |
| 692 | MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]); |
| 693 | MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]); |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 694 | MachineOperand &OpC = Root.getOperand(0); |
| 695 | |
| 696 | unsigned RegA = OpA.getReg(); |
| 697 | unsigned RegB = OpB.getReg(); |
| 698 | unsigned RegX = OpX.getReg(); |
| 699 | unsigned RegY = OpY.getReg(); |
| 700 | unsigned RegC = OpC.getReg(); |
| 701 | |
| 702 | if (TargetRegisterInfo::isVirtualRegister(RegA)) |
| 703 | MRI.constrainRegClass(RegA, RC); |
| 704 | if (TargetRegisterInfo::isVirtualRegister(RegB)) |
| 705 | MRI.constrainRegClass(RegB, RC); |
| 706 | if (TargetRegisterInfo::isVirtualRegister(RegX)) |
| 707 | MRI.constrainRegClass(RegX, RC); |
| 708 | if (TargetRegisterInfo::isVirtualRegister(RegY)) |
| 709 | MRI.constrainRegClass(RegY, RC); |
| 710 | if (TargetRegisterInfo::isVirtualRegister(RegC)) |
| 711 | MRI.constrainRegClass(RegC, RC); |
| 712 | |
| 713 | // Create a new virtual register for the result of (X op Y) instead of |
| 714 | // recycling RegB because the MachineCombiner's computation of the critical |
| 715 | // path requires a new register definition rather than an existing one. |
| 716 | unsigned NewVR = MRI.createVirtualRegister(RC); |
| 717 | InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0)); |
| 718 | |
| 719 | unsigned Opcode = Root.getOpcode(); |
| 720 | bool KillA = OpA.isKill(); |
| 721 | bool KillX = OpX.isKill(); |
| 722 | bool KillY = OpY.isKill(); |
| 723 | |
| 724 | // Create new instructions for insertion. |
| 725 | MachineInstrBuilder MIB1 = |
| 726 | BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR) |
| 727 | .addReg(RegX, getKillRegState(KillX)) |
| 728 | .addReg(RegY, getKillRegState(KillY)); |
| 729 | MachineInstrBuilder MIB2 = |
| 730 | BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC) |
| 731 | .addReg(RegA, getKillRegState(KillA)) |
| 732 | .addReg(NewVR, getKillRegState(true)); |
| 733 | |
| 734 | setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2); |
| 735 | |
| 736 | // Record new instructions for insertion and old instructions for deletion. |
| 737 | InsInstrs.push_back(MIB1); |
| 738 | InsInstrs.push_back(MIB2); |
| 739 | DelInstrs.push_back(&Prev); |
| 740 | DelInstrs.push_back(&Root); |
| 741 | } |
| 742 | |
| 743 | void TargetInstrInfo::genAlternativeCodeSequence( |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 744 | MachineInstr &Root, MachineCombinerPattern Pattern, |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 745 | SmallVectorImpl<MachineInstr *> &InsInstrs, |
| 746 | SmallVectorImpl<MachineInstr *> &DelInstrs, |
| 747 | DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const { |
| 748 | MachineRegisterInfo &MRI = Root.getParent()->getParent()->getRegInfo(); |
| 749 | |
| 750 | // Select the previous instruction in the sequence based on the input pattern. |
| 751 | MachineInstr *Prev = nullptr; |
| 752 | switch (Pattern) { |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 753 | case MachineCombinerPattern::REASSOC_AX_BY: |
| 754 | case MachineCombinerPattern::REASSOC_XA_BY: |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 755 | Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg()); |
| 756 | break; |
Sanjay Patel | 387e66e | 2015-11-05 19:34:57 +0000 | [diff] [blame] | 757 | case MachineCombinerPattern::REASSOC_AX_YB: |
| 758 | case MachineCombinerPattern::REASSOC_XA_YB: |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 759 | Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg()); |
| 760 | break; |
| 761 | default: |
| 762 | break; |
| 763 | } |
| 764 | |
| 765 | assert(Prev && "Unknown pattern for machine combiner"); |
| 766 | |
| 767 | reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg); |
Chad Rosier | 03a4730 | 2015-09-21 15:09:11 +0000 | [diff] [blame] | 768 | } |
| 769 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 770 | /// foldMemoryOperand - Same as the previous version except it allows folding |
| 771 | /// of any load and store from / to any address, not just from a specific |
| 772 | /// stack slot. |
Benjamin Kramer | f1362f6 | 2015-02-28 12:04:00 +0000 | [diff] [blame] | 773 | MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, |
| 774 | ArrayRef<unsigned> Ops, |
| 775 | MachineInstr *LoadMI) const { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 776 | assert(LoadMI->canFoldAsLoad() && "LoadMI isn't foldable!"); |
| 777 | #ifndef NDEBUG |
| 778 | for (unsigned i = 0, e = Ops.size(); i != e; ++i) |
| 779 | assert(MI->getOperand(Ops[i]).isUse() && "Folding load into def!"); |
| 780 | #endif |
| 781 | MachineBasicBlock &MBB = *MI->getParent(); |
| 782 | MachineFunction &MF = *MBB.getParent(); |
| 783 | |
| 784 | // Ask the target to do the actual folding. |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 785 | MachineInstr *NewMI = nullptr; |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 786 | int FrameIndex = 0; |
| 787 | |
| 788 | if ((MI->getOpcode() == TargetOpcode::STACKMAP || |
| 789 | MI->getOpcode() == TargetOpcode::PATCHPOINT) && |
| 790 | isLoadFromStackSlot(LoadMI, FrameIndex)) { |
| 791 | // Fold stackmap/patchpoint. |
| 792 | NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this); |
Keno Fischer | e70b31f | 2015-06-08 20:09:58 +0000 | [diff] [blame] | 793 | if (NewMI) |
| 794 | NewMI = MBB.insert(MI, NewMI); |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 795 | } else { |
| 796 | // Ask the target to do the actual folding. |
Keno Fischer | e70b31f | 2015-06-08 20:09:58 +0000 | [diff] [blame] | 797 | NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI); |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 798 | } |
Lang Hames | 3960999 | 2013-11-29 03:07:54 +0000 | [diff] [blame] | 799 | |
Craig Topper | c0196b1 | 2014-04-14 00:51:57 +0000 | [diff] [blame] | 800 | if (!NewMI) return nullptr; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 801 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 802 | // Copy the memoperands from the load to the folded instruction. |
Andrew Trick | a9f4d92 | 2013-11-14 23:45:04 +0000 | [diff] [blame] | 803 | if (MI->memoperands_empty()) { |
| 804 | NewMI->setMemRefs(LoadMI->memoperands_begin(), |
| 805 | LoadMI->memoperands_end()); |
| 806 | } |
| 807 | else { |
| 808 | // Handle the rare case of folding multiple loads. |
| 809 | NewMI->setMemRefs(MI->memoperands_begin(), |
| 810 | MI->memoperands_end()); |
| 811 | for (MachineInstr::mmo_iterator I = LoadMI->memoperands_begin(), |
| 812 | E = LoadMI->memoperands_end(); I != E; ++I) { |
| 813 | NewMI->addMemOperand(MF, *I); |
| 814 | } |
| 815 | } |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 816 | return NewMI; |
| 817 | } |
| 818 | |
| 819 | bool TargetInstrInfo:: |
| 820 | isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, |
| 821 | AliasAnalysis *AA) const { |
| 822 | const MachineFunction &MF = *MI->getParent()->getParent(); |
| 823 | const MachineRegisterInfo &MRI = MF.getRegInfo(); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 824 | |
| 825 | // Remat clients assume operand 0 is the defined register. |
| 826 | if (!MI->getNumOperands() || !MI->getOperand(0).isReg()) |
| 827 | return false; |
| 828 | unsigned DefReg = MI->getOperand(0).getReg(); |
| 829 | |
| 830 | // A sub-register definition can only be rematerialized if the instruction |
| 831 | // doesn't read the other parts of the register. Otherwise it is really a |
| 832 | // read-modify-write operation on the full virtual register which cannot be |
| 833 | // moved safely. |
| 834 | if (TargetRegisterInfo::isVirtualRegister(DefReg) && |
| 835 | MI->getOperand(0).getSubReg() && MI->readsVirtualRegister(DefReg)) |
| 836 | return false; |
| 837 | |
| 838 | // A load from a fixed stack slot can be rematerialized. This may be |
| 839 | // redundant with subsequent checks, but it's target-independent, |
| 840 | // simple, and a common case. |
| 841 | int FrameIdx = 0; |
Eric Christopher | 9d91679 | 2014-07-23 22:12:03 +0000 | [diff] [blame] | 842 | if (isLoadFromStackSlot(MI, FrameIdx) && |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 843 | MF.getFrameInfo()->isImmutableObjectIndex(FrameIdx)) |
| 844 | return true; |
| 845 | |
| 846 | // Avoid instructions obviously unsafe for remat. |
| 847 | if (MI->isNotDuplicable() || MI->mayStore() || |
| 848 | MI->hasUnmodeledSideEffects()) |
| 849 | return false; |
| 850 | |
| 851 | // Don't remat inline asm. We have no idea how expensive it is |
| 852 | // even if it's side effect free. |
| 853 | if (MI->isInlineAsm()) |
| 854 | return false; |
| 855 | |
| 856 | // Avoid instructions which load from potentially varying memory. |
| 857 | if (MI->mayLoad() && !MI->isInvariantLoad(AA)) |
| 858 | return false; |
| 859 | |
| 860 | // If any of the registers accessed are non-constant, conservatively assume |
| 861 | // the instruction is not rematerializable. |
| 862 | for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { |
| 863 | const MachineOperand &MO = MI->getOperand(i); |
| 864 | if (!MO.isReg()) continue; |
| 865 | unsigned Reg = MO.getReg(); |
| 866 | if (Reg == 0) |
| 867 | continue; |
| 868 | |
| 869 | // Check for a well-behaved physical register. |
| 870 | if (TargetRegisterInfo::isPhysicalRegister(Reg)) { |
| 871 | if (MO.isUse()) { |
| 872 | // If the physreg has no defs anywhere, it's just an ambient register |
| 873 | // and we can freely move its uses. Alternatively, if it's allocatable, |
| 874 | // it could get allocated to something with a def during allocation. |
| 875 | if (!MRI.isConstantPhysReg(Reg, MF)) |
| 876 | return false; |
| 877 | } else { |
| 878 | // A physreg def. We can't remat it. |
| 879 | return false; |
| 880 | } |
| 881 | continue; |
| 882 | } |
| 883 | |
| 884 | // Only allow one virtual-register def. There may be multiple defs of the |
| 885 | // same virtual register, though. |
| 886 | if (MO.isDef() && Reg != DefReg) |
| 887 | return false; |
| 888 | |
| 889 | // Don't allow any virtual-register uses. Rematting an instruction with |
| 890 | // virtual register uses would length the live ranges of the uses, which |
| 891 | // is not necessarily a good idea, certainly not "trivial". |
| 892 | if (MO.isUse()) |
| 893 | return false; |
| 894 | } |
| 895 | |
| 896 | // Everything checked out. |
| 897 | return true; |
| 898 | } |
| 899 | |
Michael Kuperstein | 8c65e31 | 2015-01-08 11:04:38 +0000 | [diff] [blame] | 900 | int TargetInstrInfo::getSPAdjust(const MachineInstr *MI) const { |
| 901 | const MachineFunction *MF = MI->getParent()->getParent(); |
| 902 | const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); |
| 903 | bool StackGrowsDown = |
| 904 | TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; |
| 905 | |
Matthias Braun | fa3872e | 2015-05-18 20:27:55 +0000 | [diff] [blame] | 906 | unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); |
| 907 | unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); |
Michael Kuperstein | 8c65e31 | 2015-01-08 11:04:38 +0000 | [diff] [blame] | 908 | |
| 909 | if (MI->getOpcode() != FrameSetupOpcode && |
| 910 | MI->getOpcode() != FrameDestroyOpcode) |
| 911 | return 0; |
| 912 | |
| 913 | int SPAdj = MI->getOperand(0).getImm(); |
Guozhi Wei | f66d384 | 2015-08-17 22:36:27 +0000 | [diff] [blame] | 914 | SPAdj = TFI->alignSPAdjust(SPAdj); |
Michael Kuperstein | 8c65e31 | 2015-01-08 11:04:38 +0000 | [diff] [blame] | 915 | |
| 916 | if ((!StackGrowsDown && MI->getOpcode() == FrameSetupOpcode) || |
| 917 | (StackGrowsDown && MI->getOpcode() == FrameDestroyOpcode)) |
| 918 | SPAdj = -SPAdj; |
| 919 | |
| 920 | return SPAdj; |
| 921 | } |
| 922 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 923 | /// isSchedulingBoundary - Test if the given instruction should be |
| 924 | /// considered a scheduling boundary. This primarily includes labels |
| 925 | /// and terminators. |
| 926 | bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr *MI, |
| 927 | const MachineBasicBlock *MBB, |
| 928 | const MachineFunction &MF) const { |
| 929 | // Terminators and labels can't be scheduled around. |
Rafael Espindola | b1f25f1 | 2014-03-07 06:08:31 +0000 | [diff] [blame] | 930 | if (MI->isTerminator() || MI->isPosition()) |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 931 | return true; |
| 932 | |
| 933 | // Don't attempt to schedule around any instruction that defines |
| 934 | // a stack-oriented pointer, as it's unlikely to be profitable. This |
| 935 | // saves compile time, because it doesn't require every single |
| 936 | // stack slot reference to depend on the instruction that does the |
| 937 | // modification. |
Eric Christopher | fc6de42 | 2014-08-05 02:39:49 +0000 | [diff] [blame] | 938 | const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering(); |
| 939 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
Rafael Espindola | 84921b9 | 2015-10-24 23:11:13 +0000 | [diff] [blame] | 940 | return MI->modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 941 | } |
| 942 | |
| 943 | // Provide a global flag for disabling the PreRA hazard recognizer that targets |
| 944 | // may choose to honor. |
| 945 | bool TargetInstrInfo::usePreRAHazardRecognizer() const { |
| 946 | return !DisableHazardRecognizer; |
| 947 | } |
| 948 | |
| 949 | // Default implementation of CreateTargetRAHazardRecognizer. |
| 950 | ScheduleHazardRecognizer *TargetInstrInfo:: |
Eric Christopher | f047bfd | 2014-06-13 22:38:52 +0000 | [diff] [blame] | 951 | CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 952 | const ScheduleDAG *DAG) const { |
| 953 | // Dummy hazard recognizer allows all instructions to issue. |
| 954 | return new ScheduleHazardRecognizer(); |
| 955 | } |
| 956 | |
| 957 | // Default implementation of CreateTargetMIHazardRecognizer. |
| 958 | ScheduleHazardRecognizer *TargetInstrInfo:: |
| 959 | CreateTargetMIHazardRecognizer(const InstrItineraryData *II, |
| 960 | const ScheduleDAG *DAG) const { |
| 961 | return (ScheduleHazardRecognizer *) |
| 962 | new ScoreboardHazardRecognizer(II, DAG, "misched"); |
| 963 | } |
| 964 | |
| 965 | // Default implementation of CreateTargetPostRAHazardRecognizer. |
| 966 | ScheduleHazardRecognizer *TargetInstrInfo:: |
| 967 | CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, |
| 968 | const ScheduleDAG *DAG) const { |
| 969 | return (ScheduleHazardRecognizer *) |
| 970 | new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched"); |
| 971 | } |
| 972 | |
| 973 | //===----------------------------------------------------------------------===// |
| 974 | // SelectionDAG latency interface. |
| 975 | //===----------------------------------------------------------------------===// |
| 976 | |
| 977 | int |
| 978 | TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, |
| 979 | SDNode *DefNode, unsigned DefIdx, |
| 980 | SDNode *UseNode, unsigned UseIdx) const { |
| 981 | if (!ItinData || ItinData->isEmpty()) |
| 982 | return -1; |
| 983 | |
| 984 | if (!DefNode->isMachineOpcode()) |
| 985 | return -1; |
| 986 | |
| 987 | unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass(); |
| 988 | if (!UseNode->isMachineOpcode()) |
| 989 | return ItinData->getOperandCycle(DefClass, DefIdx); |
| 990 | unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass(); |
| 991 | return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); |
| 992 | } |
| 993 | |
| 994 | int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, |
| 995 | SDNode *N) const { |
| 996 | if (!ItinData || ItinData->isEmpty()) |
| 997 | return 1; |
| 998 | |
| 999 | if (!N->isMachineOpcode()) |
| 1000 | return 1; |
| 1001 | |
| 1002 | return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass()); |
| 1003 | } |
| 1004 | |
| 1005 | //===----------------------------------------------------------------------===// |
| 1006 | // MachineInstr latency interface. |
| 1007 | //===----------------------------------------------------------------------===// |
| 1008 | |
| 1009 | unsigned |
| 1010 | TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData, |
| 1011 | const MachineInstr *MI) const { |
| 1012 | if (!ItinData || ItinData->isEmpty()) |
| 1013 | return 1; |
| 1014 | |
| 1015 | unsigned Class = MI->getDesc().getSchedClass(); |
| 1016 | int UOps = ItinData->Itineraries[Class].NumMicroOps; |
| 1017 | if (UOps >= 0) |
| 1018 | return UOps; |
| 1019 | |
| 1020 | // The # of u-ops is dynamically determined. The specific target should |
| 1021 | // override this function to return the right number. |
| 1022 | return 1; |
| 1023 | } |
| 1024 | |
| 1025 | /// Return the default expected latency for a def based on it's opcode. |
Pete Cooper | 1175945 | 2014-09-02 17:43:54 +0000 | [diff] [blame] | 1026 | unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel, |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1027 | const MachineInstr *DefMI) const { |
| 1028 | if (DefMI->isTransient()) |
| 1029 | return 0; |
| 1030 | if (DefMI->mayLoad()) |
Pete Cooper | 1175945 | 2014-09-02 17:43:54 +0000 | [diff] [blame] | 1031 | return SchedModel.LoadLatency; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1032 | if (isHighLatencyDef(DefMI->getOpcode())) |
Pete Cooper | 1175945 | 2014-09-02 17:43:54 +0000 | [diff] [blame] | 1033 | return SchedModel.HighLatency; |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1034 | return 1; |
| 1035 | } |
| 1036 | |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame^] | 1037 | unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const { |
Arnold Schwaighofer | d2f96b9 | 2013-09-30 15:28:56 +0000 | [diff] [blame] | 1038 | return 0; |
| 1039 | } |
| 1040 | |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1041 | unsigned TargetInstrInfo:: |
| 1042 | getInstrLatency(const InstrItineraryData *ItinData, |
| 1043 | const MachineInstr *MI, |
| 1044 | unsigned *PredCost) const { |
| 1045 | // Default to one cycle for no itinerary. However, an "empty" itinerary may |
| 1046 | // still have a MinLatency property, which getStageLatency checks. |
| 1047 | if (!ItinData) |
| 1048 | return MI->mayLoad() ? 2 : 1; |
| 1049 | |
| 1050 | return ItinData->getStageLatency(MI->getDesc().getSchedClass()); |
| 1051 | } |
| 1052 | |
Matthias Braun | 88e2131 | 2015-06-13 03:42:11 +0000 | [diff] [blame] | 1053 | bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel, |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1054 | const MachineInstr *DefMI, |
| 1055 | unsigned DefIdx) const { |
Matthias Braun | 88e2131 | 2015-06-13 03:42:11 +0000 | [diff] [blame] | 1056 | const InstrItineraryData *ItinData = SchedModel.getInstrItineraries(); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1057 | if (!ItinData || ItinData->isEmpty()) |
| 1058 | return false; |
| 1059 | |
| 1060 | unsigned DefClass = DefMI->getDesc().getSchedClass(); |
| 1061 | int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); |
| 1062 | return (DefCycle != -1 && DefCycle <= 1); |
| 1063 | } |
| 1064 | |
| 1065 | /// Both DefMI and UseMI must be valid. By default, call directly to the |
| 1066 | /// itinerary. This may be overriden by the target. |
| 1067 | int TargetInstrInfo:: |
| 1068 | getOperandLatency(const InstrItineraryData *ItinData, |
| 1069 | const MachineInstr *DefMI, unsigned DefIdx, |
| 1070 | const MachineInstr *UseMI, unsigned UseIdx) const { |
| 1071 | unsigned DefClass = DefMI->getDesc().getSchedClass(); |
| 1072 | unsigned UseClass = UseMI->getDesc().getSchedClass(); |
| 1073 | return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); |
| 1074 | } |
| 1075 | |
| 1076 | /// If we can determine the operand latency from the def only, without itinerary |
| 1077 | /// lookup, do so. Otherwise return -1. |
| 1078 | int TargetInstrInfo::computeDefOperandLatency( |
| 1079 | const InstrItineraryData *ItinData, |
Andrew Trick | de2109e | 2013-06-15 04:49:57 +0000 | [diff] [blame] | 1080 | const MachineInstr *DefMI) const { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1081 | |
| 1082 | // Let the target hook getInstrLatency handle missing itineraries. |
| 1083 | if (!ItinData) |
| 1084 | return getInstrLatency(ItinData, DefMI); |
| 1085 | |
Andrew Trick | de2109e | 2013-06-15 04:49:57 +0000 | [diff] [blame] | 1086 | if(ItinData->isEmpty()) |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1087 | return defaultDefLatency(ItinData->SchedModel, DefMI); |
| 1088 | |
| 1089 | // ...operand lookup required |
| 1090 | return -1; |
| 1091 | } |
| 1092 | |
| 1093 | /// computeOperandLatency - Compute and return the latency of the given data |
| 1094 | /// dependent def and use when the operand indices are already known. UseMI may |
| 1095 | /// be NULL for an unknown use. |
| 1096 | /// |
| 1097 | /// FindMin may be set to get the minimum vs. expected latency. Minimum |
| 1098 | /// latency is used for scheduling groups, while expected latency is for |
| 1099 | /// instruction cost and critical path. |
| 1100 | /// |
| 1101 | /// Depending on the subtarget's itinerary properties, this may or may not need |
| 1102 | /// to call getOperandLatency(). For most subtargets, we don't need DefIdx or |
| 1103 | /// UseIdx to compute min latency. |
| 1104 | unsigned TargetInstrInfo:: |
| 1105 | computeOperandLatency(const InstrItineraryData *ItinData, |
| 1106 | const MachineInstr *DefMI, unsigned DefIdx, |
Andrew Trick | de2109e | 2013-06-15 04:49:57 +0000 | [diff] [blame] | 1107 | const MachineInstr *UseMI, unsigned UseIdx) const { |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1108 | |
Andrew Trick | de2109e | 2013-06-15 04:49:57 +0000 | [diff] [blame] | 1109 | int DefLatency = computeDefOperandLatency(ItinData, DefMI); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1110 | if (DefLatency >= 0) |
| 1111 | return DefLatency; |
| 1112 | |
| 1113 | assert(ItinData && !ItinData->isEmpty() && "computeDefOperandLatency fail"); |
| 1114 | |
| 1115 | int OperLatency = 0; |
| 1116 | if (UseMI) |
| 1117 | OperLatency = getOperandLatency(ItinData, DefMI, DefIdx, UseMI, UseIdx); |
| 1118 | else { |
| 1119 | unsigned DefClass = DefMI->getDesc().getSchedClass(); |
| 1120 | OperLatency = ItinData->getOperandCycle(DefClass, DefIdx); |
| 1121 | } |
| 1122 | if (OperLatency >= 0) |
| 1123 | return OperLatency; |
| 1124 | |
| 1125 | // No operand latency was found. |
| 1126 | unsigned InstrLatency = getInstrLatency(ItinData, DefMI); |
| 1127 | |
| 1128 | // Expected latency is the max of the stage latency and itinerary props. |
Andrew Trick | de2109e | 2013-06-15 04:49:57 +0000 | [diff] [blame] | 1129 | InstrLatency = std::max(InstrLatency, |
| 1130 | defaultDefLatency(ItinData->SchedModel, DefMI)); |
Jakob Stoklund Olesen | c351aed | 2012-11-28 02:35:13 +0000 | [diff] [blame] | 1131 | return InstrLatency; |
| 1132 | } |
Quentin Colombet | d533cdf | 2014-08-11 22:17:14 +0000 | [diff] [blame] | 1133 | |
| 1134 | bool TargetInstrInfo::getRegSequenceInputs( |
| 1135 | const MachineInstr &MI, unsigned DefIdx, |
| 1136 | SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const { |
Quentin Colombet | 8427df9 | 2014-08-12 17:11:26 +0000 | [diff] [blame] | 1137 | assert((MI.isRegSequence() || |
| 1138 | MI.isRegSequenceLike()) && "Instruction do not have the proper type"); |
Quentin Colombet | d533cdf | 2014-08-11 22:17:14 +0000 | [diff] [blame] | 1139 | |
| 1140 | if (!MI.isRegSequence()) |
| 1141 | return getRegSequenceLikeInputs(MI, DefIdx, InputRegs); |
| 1142 | |
| 1143 | // We are looking at: |
| 1144 | // Def = REG_SEQUENCE v0, sub0, v1, sub1, ... |
| 1145 | assert(DefIdx == 0 && "REG_SEQUENCE only has one def"); |
| 1146 | for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx; |
| 1147 | OpIdx += 2) { |
| 1148 | const MachineOperand &MOReg = MI.getOperand(OpIdx); |
| 1149 | const MachineOperand &MOSubIdx = MI.getOperand(OpIdx + 1); |
| 1150 | assert(MOSubIdx.isImm() && |
| 1151 | "One of the subindex of the reg_sequence is not an immediate"); |
| 1152 | // Record Reg:SubReg, SubIdx. |
| 1153 | InputRegs.push_back(RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(), |
| 1154 | (unsigned)MOSubIdx.getImm())); |
| 1155 | } |
| 1156 | return true; |
| 1157 | } |
Quentin Colombet | 7e75cba | 2014-08-20 21:51:26 +0000 | [diff] [blame] | 1158 | |
| 1159 | bool TargetInstrInfo::getExtractSubregInputs( |
| 1160 | const MachineInstr &MI, unsigned DefIdx, |
| 1161 | RegSubRegPairAndIdx &InputReg) const { |
| 1162 | assert((MI.isExtractSubreg() || |
| 1163 | MI.isExtractSubregLike()) && "Instruction do not have the proper type"); |
| 1164 | |
| 1165 | if (!MI.isExtractSubreg()) |
| 1166 | return getExtractSubregLikeInputs(MI, DefIdx, InputReg); |
| 1167 | |
| 1168 | // We are looking at: |
| 1169 | // Def = EXTRACT_SUBREG v0.sub1, sub0. |
| 1170 | assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def"); |
| 1171 | const MachineOperand &MOReg = MI.getOperand(1); |
| 1172 | const MachineOperand &MOSubIdx = MI.getOperand(2); |
| 1173 | assert(MOSubIdx.isImm() && |
| 1174 | "The subindex of the extract_subreg is not an immediate"); |
| 1175 | |
| 1176 | InputReg.Reg = MOReg.getReg(); |
| 1177 | InputReg.SubReg = MOReg.getSubReg(); |
| 1178 | InputReg.SubIdx = (unsigned)MOSubIdx.getImm(); |
| 1179 | return true; |
| 1180 | } |
Quentin Colombet | 7e3da66 | 2014-08-20 23:49:36 +0000 | [diff] [blame] | 1181 | |
| 1182 | bool TargetInstrInfo::getInsertSubregInputs( |
| 1183 | const MachineInstr &MI, unsigned DefIdx, |
| 1184 | RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const { |
| 1185 | assert((MI.isInsertSubreg() || |
| 1186 | MI.isInsertSubregLike()) && "Instruction do not have the proper type"); |
| 1187 | |
| 1188 | if (!MI.isInsertSubreg()) |
| 1189 | return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg); |
| 1190 | |
| 1191 | // We are looking at: |
| 1192 | // Def = INSERT_SEQUENCE v0, v1, sub0. |
| 1193 | assert(DefIdx == 0 && "INSERT_SUBREG only has one def"); |
| 1194 | const MachineOperand &MOBaseReg = MI.getOperand(1); |
| 1195 | const MachineOperand &MOInsertedReg = MI.getOperand(2); |
| 1196 | const MachineOperand &MOSubIdx = MI.getOperand(3); |
| 1197 | assert(MOSubIdx.isImm() && |
| 1198 | "One of the subindex of the reg_sequence is not an immediate"); |
| 1199 | BaseReg.Reg = MOBaseReg.getReg(); |
| 1200 | BaseReg.SubReg = MOBaseReg.getSubReg(); |
| 1201 | |
| 1202 | InsertedReg.Reg = MOInsertedReg.getReg(); |
| 1203 | InsertedReg.SubReg = MOInsertedReg.getSubReg(); |
| 1204 | InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm(); |
| 1205 | return true; |
| 1206 | } |