Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 1 | //===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===// |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This implements a simple VLIW packetizer using DFA. The packetizer works on |
| 11 | // machine basic blocks. For each instruction I in BB, the packetizer consults |
| 12 | // the DFA to see if machine resources are available to execute I. If so, the |
| 13 | // packetizer checks if I depends on any instruction J in the current packet. |
| 14 | // If no dependency is found, I is added to current packet and machine resource |
| 15 | // is marked as taken. If any dependency is found, a target API call is made to |
| 16 | // prune the dependence. |
| 17 | // |
| 18 | //===----------------------------------------------------------------------===// |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 19 | |
Chandler Carruth | 6bda14b | 2017-06-06 11:49:48 +0000 | [diff] [blame] | 20 | #include "HexagonVLIWPacketizer.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 21 | #include "Hexagon.h" |
| 22 | #include "HexagonInstrInfo.h" |
Jyotsna Verma | 8425643 | 2013-03-01 17:37:13 +0000 | [diff] [blame] | 23 | #include "HexagonRegisterInfo.h" |
| 24 | #include "HexagonSubtarget.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 25 | #include "llvm/ADT/BitVector.h" |
| 26 | #include "llvm/ADT/DenseSet.h" |
| 27 | #include "llvm/ADT/STLExtras.h" |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 28 | #include "llvm/Analysis/AliasAnalysis.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 29 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 30 | #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" |
Chandler Carruth | 8a8cd2b | 2014-01-07 11:48:04 +0000 | [diff] [blame] | 31 | #include "llvm/CodeGen/MachineDominators.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 32 | #include "llvm/CodeGen/MachineFrameInfo.h" |
| 33 | #include "llvm/CodeGen/MachineFunction.h" |
Chandler Carruth | 8a8cd2b | 2014-01-07 11:48:04 +0000 | [diff] [blame] | 34 | #include "llvm/CodeGen/MachineFunctionPass.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 35 | #include "llvm/CodeGen/MachineInstr.h" |
Matthias Braun | f842297 | 2017-12-13 02:51:04 +0000 | [diff] [blame] | 36 | #include "llvm/CodeGen/MachineInstrBundle.h" |
Chandler Carruth | 8a8cd2b | 2014-01-07 11:48:04 +0000 | [diff] [blame] | 37 | #include "llvm/CodeGen/MachineLoopInfo.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 38 | #include "llvm/CodeGen/MachineOperand.h" |
| 39 | #include "llvm/CodeGen/ScheduleDAG.h" |
David Blaikie | b3bde2e | 2017-11-17 01:07:10 +0000 | [diff] [blame] | 40 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
| 41 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 42 | #include "llvm/IR/DebugLoc.h" |
| 43 | #include "llvm/MC/MCInstrDesc.h" |
| 44 | #include "llvm/Pass.h" |
Chandler Carruth | 8a8cd2b | 2014-01-07 11:48:04 +0000 | [diff] [blame] | 45 | #include "llvm/Support/CommandLine.h" |
Chandler Carruth | 8a8cd2b | 2014-01-07 11:48:04 +0000 | [diff] [blame] | 46 | #include "llvm/Support/Debug.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 47 | #include "llvm/Support/ErrorHandling.h" |
| 48 | #include "llvm/Support/raw_ostream.h" |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 49 | #include <cassert> |
| 50 | #include <cstdint> |
| 51 | #include <iterator> |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 52 | |
| 53 | using namespace llvm; |
| 54 | |
Chandler Carruth | 84e68b2 | 2014-04-22 02:41:26 +0000 | [diff] [blame] | 55 | #define DEBUG_TYPE "packets" |
| 56 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 57 | static cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden, |
| 58 | cl::ZeroOrMore, cl::init(false), |
| 59 | cl::desc("Disable Hexagon packetizer pass")); |
| 60 | |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 61 | cl::opt<bool> Slot1Store("slot1-store-slot0-load", cl::Hidden, |
| 62 | cl::ZeroOrMore, cl::init(true), |
| 63 | cl::desc("Allow slot1 store and slot0 load")); |
| 64 | |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 65 | static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles", |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 66 | cl::ZeroOrMore, cl::Hidden, cl::init(true), |
| 67 | cl::desc("Allow non-solo packetization of volatile memory references")); |
| 68 | |
| 69 | static cl::opt<bool> EnableGenAllInsnClass("enable-gen-insn", cl::init(false), |
| 70 | cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC")); |
| 71 | |
| 72 | static cl::opt<bool> DisableVecDblNVStores("disable-vecdbl-nv-stores", |
| 73 | cl::init(false), cl::Hidden, cl::ZeroOrMore, |
| 74 | cl::desc("Disable vector double new-value-stores")); |
| 75 | |
| 76 | extern cl::opt<bool> ScheduleInlineAsm; |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 77 | |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 78 | namespace llvm { |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 79 | |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 80 | FunctionPass *createHexagonPacketizer(); |
| 81 | void initializeHexagonPacketizerPass(PassRegistry&); |
| 82 | |
| 83 | } // end namespace llvm |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 84 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 85 | namespace { |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 86 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 87 | class HexagonPacketizer : public MachineFunctionPass { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 88 | public: |
| 89 | static char ID; |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 90 | |
Krzysztof Parzyszek | df4a05d | 2017-07-10 18:38:52 +0000 | [diff] [blame] | 91 | HexagonPacketizer() : MachineFunctionPass(ID) {} |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 92 | |
Craig Topper | 906c2cd | 2014-04-29 07:58:16 +0000 | [diff] [blame] | 93 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 94 | AU.setPreservesCFG(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 95 | AU.addRequired<AAResultsWrapperPass>(); |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 96 | AU.addRequired<MachineBranchProbabilityInfo>(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 97 | AU.addRequired<MachineDominatorTree>(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 98 | AU.addRequired<MachineLoopInfo>(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 99 | AU.addPreserved<MachineDominatorTree>(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 100 | AU.addPreserved<MachineLoopInfo>(); |
| 101 | MachineFunctionPass::getAnalysisUsage(AU); |
| 102 | } |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 103 | |
Mehdi Amini | 117296c | 2016-10-01 02:56:57 +0000 | [diff] [blame] | 104 | StringRef getPassName() const override { return "Hexagon Packetizer"; } |
Craig Topper | 906c2cd | 2014-04-29 07:58:16 +0000 | [diff] [blame] | 105 | bool runOnMachineFunction(MachineFunction &Fn) override; |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 106 | |
Derek Schuff | 1dbf7a5 | 2016-04-04 17:09:25 +0000 | [diff] [blame] | 107 | MachineFunctionProperties getRequiredProperties() const override { |
| 108 | return MachineFunctionProperties().set( |
Matthias Braun | 1eb4736 | 2016-08-25 01:27:13 +0000 | [diff] [blame] | 109 | MachineFunctionProperties::Property::NoVRegs); |
Derek Schuff | 1dbf7a5 | 2016-04-04 17:09:25 +0000 | [diff] [blame] | 110 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 111 | |
| 112 | private: |
| 113 | const HexagonInstrInfo *HII; |
| 114 | const HexagonRegisterInfo *HRI; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 115 | }; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 116 | |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 117 | } // end anonymous namespace |
| 118 | |
| 119 | char HexagonPacketizer::ID = 0; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 120 | |
Krzysztof Parzyszek | df4a05d | 2017-07-10 18:38:52 +0000 | [diff] [blame] | 121 | INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer", |
| 122 | "Hexagon Packetizer", false, false) |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 123 | INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) |
| 124 | INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo) |
| 125 | INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) |
Chandler Carruth | 7b560d4 | 2015-09-09 17:55:00 +0000 | [diff] [blame] | 126 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) |
Krzysztof Parzyszek | df4a05d | 2017-07-10 18:38:52 +0000 | [diff] [blame] | 127 | INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer", |
| 128 | "Hexagon Packetizer", false, false) |
Jyotsna Verma | 1d29750 | 2013-05-02 15:39:30 +0000 | [diff] [blame] | 129 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 130 | HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF, |
| 131 | MachineLoopInfo &MLI, AliasAnalysis *AA, |
| 132 | const MachineBranchProbabilityInfo *MBPI) |
| 133 | : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI) { |
| 134 | HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo(); |
| 135 | HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo(); |
Krzysztof Parzyszek | 9be6673 | 2016-07-15 17:48:09 +0000 | [diff] [blame] | 136 | |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 137 | addMutation(llvm::make_unique<HexagonSubtarget::UsrOverflowMutation>()); |
| 138 | addMutation(llvm::make_unique<HexagonSubtarget::HVXMemLatencyMutation>()); |
| 139 | addMutation(llvm::make_unique<HexagonSubtarget::BankConflictMutation>()); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 140 | } |
| 141 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 142 | // Check if FirstI modifies a register that SecondI reads. |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 143 | static bool hasWriteToReadDep(const MachineInstr &FirstI, |
| 144 | const MachineInstr &SecondI, |
| 145 | const TargetRegisterInfo *TRI) { |
| 146 | for (auto &MO : FirstI.operands()) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 147 | if (!MO.isReg() || !MO.isDef()) |
| 148 | continue; |
| 149 | unsigned R = MO.getReg(); |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 150 | if (SecondI.readsRegister(R, TRI)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 151 | return true; |
| 152 | } |
| 153 | return false; |
| 154 | } |
| 155 | |
| 156 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 157 | static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 158 | MachineBasicBlock::iterator BundleIt, bool Before) { |
| 159 | MachineBasicBlock::instr_iterator InsertPt; |
| 160 | if (Before) |
Duncan P. N. Exon Smith | d84f600 | 2016-02-22 21:30:15 +0000 | [diff] [blame] | 161 | InsertPt = BundleIt.getInstrIterator(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 162 | else |
Duncan P. N. Exon Smith | d84f600 | 2016-02-22 21:30:15 +0000 | [diff] [blame] | 163 | InsertPt = std::next(BundleIt).getInstrIterator(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 164 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 165 | MachineBasicBlock &B = *MI.getParent(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 166 | // The instruction should at least be bundled with the preceding instruction |
| 167 | // (there will always be one, i.e. BUNDLE, if nothing else). |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 168 | assert(MI.isBundledWithPred()); |
| 169 | if (MI.isBundledWithSucc()) { |
| 170 | MI.clearFlag(MachineInstr::BundledSucc); |
| 171 | MI.clearFlag(MachineInstr::BundledPred); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 172 | } else { |
| 173 | // If it's not bundled with the successor (i.e. it is the last one |
| 174 | // in the bundle), then we can simply unbundle it from the predecessor, |
| 175 | // which will take care of updating the predecessor's flag. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 176 | MI.unbundleFromPred(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 177 | } |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 178 | B.splice(InsertPt, &B, MI.getIterator()); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 179 | |
| 180 | // Get the size of the bundle without asserting. |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 181 | MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 182 | MachineBasicBlock::const_instr_iterator E = B.instr_end(); |
| 183 | unsigned Size = 0; |
| 184 | for (++I; I != E && I->isBundledWithPred(); ++I) |
| 185 | ++Size; |
| 186 | |
| 187 | // If there are still two or more instructions, then there is nothing |
| 188 | // else to be done. |
| 189 | if (Size > 1) |
| 190 | return BundleIt; |
| 191 | |
| 192 | // Otherwise, extract the single instruction out and delete the bundle. |
| 193 | MachineBasicBlock::iterator NextIt = std::next(BundleIt); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 194 | MachineInstr &SingleI = *BundleIt->getNextNode(); |
| 195 | SingleI.unbundleFromPred(); |
| 196 | assert(!SingleI.isBundledWithSucc()); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 197 | BundleIt->eraseFromParent(); |
| 198 | return NextIt; |
| 199 | } |
| 200 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 201 | bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) { |
Krzysztof Parzyszek | 5d41cc1 | 2018-03-12 17:47:46 +0000 | [diff] [blame] | 202 | auto &HST = MF.getSubtarget<HexagonSubtarget>(); |
| 203 | if (DisablePacketizer || !HST.usePackets() || skipFunction(MF.getFunction())) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 204 | return false; |
| 205 | |
Krzysztof Parzyszek | 5d41cc1 | 2018-03-12 17:47:46 +0000 | [diff] [blame] | 206 | HII = HST.getInstrInfo(); |
| 207 | HRI = HST.getRegisterInfo(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 208 | auto &MLI = getAnalysis<MachineLoopInfo>(); |
| 209 | auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); |
| 210 | auto *MBPI = &getAnalysis<MachineBranchProbabilityInfo>(); |
| 211 | |
| 212 | if (EnableGenAllInsnClass) |
| 213 | HII->genAllInsnTimingClasses(MF); |
| 214 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 215 | // Instantiate the packetizer. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 216 | HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 217 | |
| 218 | // DFA state table should not be empty. |
| 219 | assert(Packetizer.getResourceTracker() && "Empty DFA table!"); |
| 220 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 221 | // Loop over all basic blocks and remove KILL pseudo-instructions |
| 222 | // These instructions confuse the dependence analysis. Consider: |
| 223 | // D0 = ... (Insn 0) |
| 224 | // R0 = KILL R0, D0 (Insn 1) |
| 225 | // R0 = ... (Insn 2) |
| 226 | // Here, Insn 1 will result in the dependence graph not emitting an output |
| 227 | // dependence between Insn 0 and Insn 2. This can lead to incorrect |
| 228 | // packetization |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 229 | for (auto &MB : MF) { |
| 230 | auto End = MB.end(); |
| 231 | auto MI = MB.begin(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 232 | while (MI != End) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 233 | auto NextI = std::next(MI); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 234 | if (MI->isKill()) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 235 | MB.erase(MI); |
| 236 | End = MB.end(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 237 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 238 | MI = NextI; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 239 | } |
| 240 | } |
| 241 | |
| 242 | // Loop over all of the basic blocks. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 243 | for (auto &MB : MF) { |
| 244 | auto Begin = MB.begin(), End = MB.end(); |
| 245 | while (Begin != End) { |
Krzysztof Parzyszek | e3ec97b | 2017-05-24 13:43:42 +0000 | [diff] [blame] | 246 | // Find the first non-boundary starting from the end of the last |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 247 | // scheduling region. |
| 248 | MachineBasicBlock::iterator RB = Begin; |
Duncan P. N. Exon Smith | 9cfc75c | 2016-06-30 00:01:54 +0000 | [diff] [blame] | 249 | while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 250 | ++RB; |
Krzysztof Parzyszek | e3ec97b | 2017-05-24 13:43:42 +0000 | [diff] [blame] | 251 | // Find the first boundary starting from the beginning of the new |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 252 | // region. |
| 253 | MachineBasicBlock::iterator RE = RB; |
Duncan P. N. Exon Smith | 9cfc75c | 2016-06-30 00:01:54 +0000 | [diff] [blame] | 254 | while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 255 | ++RE; |
| 256 | // Add the scheduling boundary if it's not block end. |
| 257 | if (RE != End) |
| 258 | ++RE; |
| 259 | // If RB == End, then RE == End. |
| 260 | if (RB != End) |
| 261 | Packetizer.PacketizeMIs(&MB, RB, RE); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 262 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 263 | Begin = RE; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 264 | } |
| 265 | } |
| 266 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 267 | Packetizer.unpacketizeSoloInstrs(MF); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 268 | return true; |
| 269 | } |
| 270 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 271 | // Reserve resources for a constant extender. Trigger an assertion if the |
| 272 | // reservation fails. |
| 273 | void HexagonPacketizerList::reserveResourcesForConstExt() { |
| 274 | if (!tryAllocateResourcesForConstExt(true)) |
| 275 | llvm_unreachable("Resources not available"); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 276 | } |
| 277 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 278 | bool HexagonPacketizerList::canReserveResourcesForConstExt() { |
| 279 | return tryAllocateResourcesForConstExt(false); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 280 | } |
| 281 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 282 | // Allocate resources (i.e. 4 bytes) for constant extender. If succeeded, |
| 283 | // return true, otherwise, return false. |
| 284 | bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) { |
| 285 | auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc()); |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 286 | bool Avail = ResourceTracker->canReserveResources(*ExtMI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 287 | if (Reserve && Avail) |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 288 | ResourceTracker->reserveResources(*ExtMI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 289 | MF.DeleteMachineInstr(ExtMI); |
| 290 | return Avail; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 291 | } |
| 292 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 293 | bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 294 | SDep::Kind DepType, unsigned DepReg) { |
| 295 | // Check for LR dependence. |
| 296 | if (DepReg == HRI->getRARegister()) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 297 | return true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 298 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 299 | if (HII->isDeallocRet(MI)) |
| 300 | if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister()) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 301 | return true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 302 | |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 303 | // Call-like instructions can be packetized with preceding instructions |
| 304 | // that define registers implicitly used or modified by the call. Explicit |
| 305 | // uses are still prohibited, as in the case of indirect calls: |
| 306 | // r0 = ... |
| 307 | // J2_jumpr r0 |
| 308 | if (DepType == SDep::Data) { |
| 309 | for (const MachineOperand MO : MI.operands()) |
| 310 | if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit()) |
| 311 | return true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 312 | } |
| 313 | |
| 314 | return false; |
| 315 | } |
| 316 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 317 | static bool isRegDependence(const SDep::Kind DepType) { |
| 318 | return DepType == SDep::Data || DepType == SDep::Anti || |
| 319 | DepType == SDep::Output; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 320 | } |
| 321 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 322 | static bool isDirectJump(const MachineInstr &MI) { |
| 323 | return MI.getOpcode() == Hexagon::J2_jump; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 324 | } |
| 325 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 326 | static bool isSchedBarrier(const MachineInstr &MI) { |
| 327 | switch (MI.getOpcode()) { |
Colin LeMahieu | b882f2b | 2015-02-05 18:56:28 +0000 | [diff] [blame] | 328 | case Hexagon::Y2_barrier: |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 329 | return true; |
| 330 | } |
| 331 | return false; |
| 332 | } |
| 333 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 334 | static bool isControlFlow(const MachineInstr &MI) { |
| 335 | return MI.getDesc().isTerminator() || MI.getDesc().isCall(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 336 | } |
| 337 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 338 | /// Returns true if the instruction modifies a callee-saved register. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 339 | static bool doesModifyCalleeSavedReg(const MachineInstr &MI, |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 340 | const TargetRegisterInfo *TRI) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 341 | const MachineFunction &MF = *MI.getParent()->getParent(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 342 | for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR) |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 343 | if (MI.modifiesRegister(*CSR, TRI)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 344 | return true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 345 | return false; |
| 346 | } |
| 347 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 348 | // Returns true if an instruction can be promoted to .new predicate or |
| 349 | // new-value store. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 350 | bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI, |
Krzysztof Parzyszek | 2a48059 | 2016-07-26 20:30:30 +0000 | [diff] [blame] | 351 | const TargetRegisterClass *NewRC) { |
| 352 | // Vector stores can be predicated, and can be new-value stores, but |
| 353 | // they cannot be predicated on a .new predicate value. |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 354 | if (NewRC == &Hexagon::PredRegsRegClass) { |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 355 | if (HII->isHVXVec(MI) && MI.mayStore()) |
Krzysztof Parzyszek | 2a48059 | 2016-07-26 20:30:30 +0000 | [diff] [blame] | 356 | return false; |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 357 | return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, nullptr) > 0; |
| 358 | } |
| 359 | // If the class is not PredRegs, it could only apply to new-value stores. |
| 360 | return HII->mayBeNewStore(MI); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 361 | } |
| 362 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 363 | // Promote an instructiont to its .cur form. |
| 364 | // At this time, we have already made a call to canPromoteToDotCur and made |
| 365 | // sure that it can *indeed* be promoted. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 366 | bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 367 | SDep::Kind DepType, MachineBasicBlock::iterator &MII, |
| 368 | const TargetRegisterClass* RC) { |
| 369 | assert(DepType == SDep::Data); |
| 370 | int CurOpcode = HII->getDotCurOp(MI); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 371 | MI.setDesc(HII->get(CurOpcode)); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 372 | return true; |
| 373 | } |
| 374 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 375 | void HexagonPacketizerList::cleanUpDotCur() { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 376 | MachineInstr *MI = nullptr; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 377 | for (auto BI : CurrentPacketMIs) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 378 | LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump();); |
Krzysztof Parzyszek | 0a8043e | 2017-05-03 15:28:56 +0000 | [diff] [blame] | 379 | if (HII->isDotCurInst(*BI)) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 380 | MI = BI; |
| 381 | continue; |
| 382 | } |
| 383 | if (MI) { |
| 384 | for (auto &MO : BI->operands()) |
| 385 | if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg()) |
| 386 | return; |
| 387 | } |
| 388 | } |
| 389 | if (!MI) |
| 390 | return; |
| 391 | // We did not find a use of the CUR, so de-cur it. |
Krzysztof Parzyszek | 0a8043e | 2017-05-03 15:28:56 +0000 | [diff] [blame] | 392 | MI->setDesc(HII->get(HII->getNonDotCurOp(*MI))); |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 393 | LLVM_DEBUG(dbgs() << "Demoted CUR "; MI->dump();); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 394 | } |
| 395 | |
| 396 | // Check to see if an instruction can be dot cur. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 397 | bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 398 | const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, |
| 399 | const TargetRegisterClass *RC) { |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 400 | if (!HII->isHVXVec(MI)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 401 | return false; |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 402 | if (!HII->isHVXVec(*MII)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 403 | return false; |
| 404 | |
| 405 | // Already a dot new instruction. |
| 406 | if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI)) |
| 407 | return false; |
| 408 | |
| 409 | if (!HII->mayBeCurLoad(MI)) |
| 410 | return false; |
| 411 | |
| 412 | // The "cur value" cannot come from inline asm. |
| 413 | if (PacketSU->getInstr()->isInlineAsm()) |
| 414 | return false; |
| 415 | |
| 416 | // Make sure candidate instruction uses cur. |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 417 | LLVM_DEBUG(dbgs() << "Can we DOT Cur Vector MI\n"; MI.dump(); |
| 418 | dbgs() << "in packet\n";); |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 419 | MachineInstr &MJ = *MII; |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 420 | LLVM_DEBUG({ |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 421 | dbgs() << "Checking CUR against "; |
| 422 | MJ.dump(); |
| 423 | }); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 424 | unsigned DestReg = MI.getOperand(0).getReg(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 425 | bool FoundMatch = false; |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 426 | for (auto &MO : MJ.operands()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 427 | if (MO.isReg() && MO.getReg() == DestReg) |
| 428 | FoundMatch = true; |
| 429 | if (!FoundMatch) |
| 430 | return false; |
| 431 | |
| 432 | // Check for existing uses of a vector register within the packet which |
| 433 | // would be affected by converting a vector load into .cur formt. |
| 434 | for (auto BI : CurrentPacketMIs) { |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 435 | LLVM_DEBUG(dbgs() << "packet has "; BI->dump();); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 436 | if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo())) |
| 437 | return false; |
| 438 | } |
| 439 | |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 440 | LLVM_DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump();); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 441 | // We can convert the opcode into a .cur. |
| 442 | return true; |
| 443 | } |
| 444 | |
| 445 | // Promote an instruction to its .new form. At this time, we have already |
| 446 | // made a call to canPromoteToDotNew and made sure that it can *indeed* be |
| 447 | // promoted. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 448 | bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 449 | SDep::Kind DepType, MachineBasicBlock::iterator &MII, |
| 450 | const TargetRegisterClass* RC) { |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 451 | assert(DepType == SDep::Data); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 452 | int NewOpcode; |
| 453 | if (RC == &Hexagon::PredRegsRegClass) |
| 454 | NewOpcode = HII->getDotNewPredOp(MI, MBPI); |
| 455 | else |
| 456 | NewOpcode = HII->getDotNewOp(MI); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 457 | MI.setDesc(HII->get(NewOpcode)); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 458 | return true; |
| 459 | } |
| 460 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 461 | bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) { |
Krzysztof Parzyszek | 143158b | 2017-03-06 17:03:16 +0000 | [diff] [blame] | 462 | int NewOpcode = HII->getDotOldOp(MI); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 463 | MI.setDesc(HII->get(NewOpcode)); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 464 | return true; |
| 465 | } |
| 466 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 467 | bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) { |
| 468 | unsigned Opc = MI.getOpcode(); |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 469 | switch (Opc) { |
| 470 | case Hexagon::S2_storerd_io: |
| 471 | case Hexagon::S2_storeri_io: |
| 472 | case Hexagon::S2_storerh_io: |
| 473 | case Hexagon::S2_storerb_io: |
| 474 | break; |
| 475 | default: |
| 476 | llvm_unreachable("Unexpected instruction"); |
| 477 | } |
Matthias Braun | 941a705 | 2016-07-28 18:40:00 +0000 | [diff] [blame] | 478 | unsigned FrameSize = MF.getFrameInfo().getStackSize(); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 479 | MachineOperand &Off = MI.getOperand(1); |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 480 | int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE); |
Krzysztof Parzyszek | 5577297 | 2017-09-15 15:46:05 +0000 | [diff] [blame] | 481 | if (HII->isValidOffset(Opc, NewOff, HRI)) { |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 482 | Off.setImm(NewOff); |
| 483 | return true; |
| 484 | } |
| 485 | return false; |
| 486 | } |
| 487 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 488 | void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) { |
| 489 | unsigned Opc = MI.getOpcode(); |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 490 | switch (Opc) { |
| 491 | case Hexagon::S2_storerd_io: |
| 492 | case Hexagon::S2_storeri_io: |
| 493 | case Hexagon::S2_storerh_io: |
| 494 | case Hexagon::S2_storerb_io: |
| 495 | break; |
| 496 | default: |
| 497 | llvm_unreachable("Unexpected instruction"); |
| 498 | } |
Matthias Braun | 941a705 | 2016-07-28 18:40:00 +0000 | [diff] [blame] | 499 | unsigned FrameSize = MF.getFrameInfo().getStackSize(); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 500 | MachineOperand &Off = MI.getOperand(1); |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 501 | Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE); |
| 502 | } |
| 503 | |
Krzysztof Parzyszek | 8f174dd | 2017-10-11 15:51:44 +0000 | [diff] [blame] | 504 | /// Return true if we can update the offset in MI so that MI and MJ |
| 505 | /// can be packetized together. |
| 506 | bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) { |
| 507 | assert(SUI->getInstr() && SUJ->getInstr()); |
| 508 | MachineInstr &MI = *SUI->getInstr(); |
| 509 | MachineInstr &MJ = *SUJ->getInstr(); |
| 510 | |
| 511 | unsigned BPI, OPI; |
| 512 | if (!HII->getBaseAndOffsetPosition(MI, BPI, OPI)) |
| 513 | return false; |
| 514 | unsigned BPJ, OPJ; |
| 515 | if (!HII->getBaseAndOffsetPosition(MJ, BPJ, OPJ)) |
| 516 | return false; |
| 517 | unsigned Reg = MI.getOperand(BPI).getReg(); |
| 518 | if (Reg != MJ.getOperand(BPJ).getReg()) |
| 519 | return false; |
| 520 | // Make sure that the dependences do not restrict adding MI to the packet. |
| 521 | // That is, ignore anti dependences, and make sure the only data dependence |
| 522 | // involves the specific register. |
| 523 | for (const auto &PI : SUI->Preds) |
| 524 | if (PI.getKind() != SDep::Anti && |
| 525 | (PI.getKind() != SDep::Data || PI.getReg() != Reg)) |
| 526 | return false; |
| 527 | int Incr; |
| 528 | if (!HII->getIncrementValue(MJ, Incr)) |
| 529 | return false; |
| 530 | |
| 531 | int64_t Offset = MI.getOperand(OPI).getImm(); |
Krzysztof Parzyszek | 0f983d6 | 2018-03-30 19:28:37 +0000 | [diff] [blame] | 532 | if (!HII->isValidOffset(MI.getOpcode(), Offset+Incr, HRI)) |
| 533 | return false; |
| 534 | |
Krzysztof Parzyszek | 8f174dd | 2017-10-11 15:51:44 +0000 | [diff] [blame] | 535 | MI.getOperand(OPI).setImm(Offset + Incr); |
| 536 | ChangedOffset = Offset; |
| 537 | return true; |
| 538 | } |
| 539 | |
| 540 | /// Undo the changed offset. This is needed if the instruction cannot be |
| 541 | /// added to the current packet due to a different instruction. |
| 542 | void HexagonPacketizerList::undoChangedOffset(MachineInstr &MI) { |
| 543 | unsigned BP, OP; |
| 544 | if (!HII->getBaseAndOffsetPosition(MI, BP, OP)) |
| 545 | llvm_unreachable("Unable to find base and offset operands."); |
| 546 | MI.getOperand(OP).setImm(ChangedOffset); |
| 547 | } |
| 548 | |
Jyotsna Verma | 300f0b9 | 2013-05-10 20:27:34 +0000 | [diff] [blame] | 549 | enum PredicateKind { |
| 550 | PK_False, |
| 551 | PK_True, |
| 552 | PK_Unknown |
| 553 | }; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 554 | |
Jyotsna Verma | 300f0b9 | 2013-05-10 20:27:34 +0000 | [diff] [blame] | 555 | /// Returns true if an instruction is predicated on p0 and false if it's |
| 556 | /// predicated on !p0. |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame] | 557 | static PredicateKind getPredicateSense(const MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 558 | const HexagonInstrInfo *HII) { |
| 559 | if (!HII->isPredicated(MI)) |
Jyotsna Verma | 300f0b9 | 2013-05-10 20:27:34 +0000 | [diff] [blame] | 560 | return PK_Unknown; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 561 | if (HII->isPredicatedTrue(MI)) |
Jyotsna Verma | 300f0b9 | 2013-05-10 20:27:34 +0000 | [diff] [blame] | 562 | return PK_True; |
Jyotsna Verma | 300f0b9 | 2013-05-10 20:27:34 +0000 | [diff] [blame] | 563 | return PK_False; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 564 | } |
| 565 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 566 | static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 567 | const HexagonInstrInfo *HII) { |
Krzysztof Parzyszek | 8fb181c | 2016-08-01 17:55:48 +0000 | [diff] [blame] | 568 | assert(HII->isPostIncrement(MI) && "Not a post increment operation."); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 569 | #ifndef NDEBUG |
| 570 | // Post Increment means duplicates. Use dense map to find duplicates in the |
| 571 | // list. Caution: Densemap initializes with the minimum of 64 buckets, |
| 572 | // whereas there are at most 5 operands in the post increment. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 573 | DenseSet<unsigned> DefRegsSet; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 574 | for (auto &MO : MI.operands()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 575 | if (MO.isReg() && MO.isDef()) |
| 576 | DefRegsSet.insert(MO.getReg()); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 577 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 578 | for (auto &MO : MI.operands()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 579 | if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg())) |
| 580 | return MO; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 581 | #else |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 582 | if (MI.mayLoad()) { |
| 583 | const MachineOperand &Op1 = MI.getOperand(1); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 584 | // The 2nd operand is always the post increment operand in load. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 585 | assert(Op1.isReg() && "Post increment operand has be to a register."); |
| 586 | return Op1; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 587 | } |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 588 | if (MI.getDesc().mayStore()) { |
| 589 | const MachineOperand &Op0 = MI.getOperand(0); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 590 | // The 1st operand is always the post increment operand in store. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 591 | assert(Op0.isReg() && "Post increment operand has be to a register."); |
| 592 | return Op0; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 593 | } |
| 594 | #endif |
| 595 | // we should never come here. |
| 596 | llvm_unreachable("mayLoad or mayStore not set for Post Increment operation"); |
| 597 | } |
| 598 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 599 | // Get the value being stored. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 600 | static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 601 | // value being stored is always the last operand. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 602 | return MI.getOperand(MI.getNumOperands()-1); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 603 | } |
| 604 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 605 | static bool isLoadAbsSet(const MachineInstr &MI) { |
| 606 | unsigned Opc = MI.getOpcode(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 607 | switch (Opc) { |
| 608 | case Hexagon::L4_loadrd_ap: |
| 609 | case Hexagon::L4_loadrb_ap: |
| 610 | case Hexagon::L4_loadrh_ap: |
| 611 | case Hexagon::L4_loadrub_ap: |
| 612 | case Hexagon::L4_loadruh_ap: |
| 613 | case Hexagon::L4_loadri_ap: |
| 614 | return true; |
| 615 | } |
| 616 | return false; |
| 617 | } |
| 618 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 619 | static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 620 | assert(isLoadAbsSet(MI)); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 621 | return MI.getOperand(1); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 622 | } |
| 623 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 624 | // Can be new value store? |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 625 | // Following restrictions are to be respected in convert a store into |
| 626 | // a new value store. |
| 627 | // 1. If an instruction uses auto-increment, its address register cannot |
| 628 | // be a new-value register. Arch Spec 5.4.2.1 |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 629 | // 2. If an instruction uses absolute-set addressing mode, its address |
| 630 | // register cannot be a new-value register. Arch Spec 5.4.2.1. |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 631 | // 3. If an instruction produces a 64-bit result, its registers cannot be used |
| 632 | // as new-value registers. Arch Spec 5.4.2.2. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 633 | // 4. If the instruction that sets the new-value register is conditional, then |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 634 | // the instruction that uses the new-value register must also be conditional, |
| 635 | // and both must always have their predicates evaluate identically. |
| 636 | // Arch Spec 5.4.2.3. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 637 | // 5. There is an implied restriction that a packet cannot have another store, |
| 638 | // if there is a new value store in the packet. Corollary: if there is |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 639 | // already a store in a packet, there can not be a new value store. |
| 640 | // Arch Spec: 3.4.4.2 |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 641 | bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI, |
| 642 | const MachineInstr &PacketMI, unsigned DepReg) { |
Jyotsna Verma | 438cec5 | 2013-05-10 20:58:11 +0000 | [diff] [blame] | 643 | // Make sure we are looking at the store, that can be promoted. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 644 | if (!HII->mayBeNewStore(MI)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 645 | return false; |
| 646 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 647 | // Make sure there is dependency and can be new value'd. |
| 648 | const MachineOperand &Val = getStoreValueOperand(MI); |
| 649 | if (Val.isReg() && Val.getReg() != DepReg) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 650 | return false; |
| 651 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 652 | const MCInstrDesc& MCID = PacketMI.getDesc(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 653 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 654 | // First operand is always the result. |
| 655 | const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF); |
| 656 | // Double regs can not feed into new value store: PRM section: 5.4.2.2. |
| 657 | if (PacketRC == &Hexagon::DoubleRegsRegClass) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 658 | return false; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 659 | |
| 660 | // New-value stores are of class NV (slot 0), dual stores require class ST |
| 661 | // in slot 0 (PRM 5.5). |
| 662 | for (auto I : CurrentPacketMIs) { |
| 663 | SUnit *PacketSU = MIToSUnit.find(I)->second; |
| 664 | if (PacketSU->getInstr()->mayStore()) |
| 665 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 666 | } |
| 667 | |
| 668 | // Make sure it's NOT the post increment register that we are going to |
| 669 | // new value. |
Krzysztof Parzyszek | 8fb181c | 2016-08-01 17:55:48 +0000 | [diff] [blame] | 670 | if (HII->isPostIncrement(MI) && |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 671 | getPostIncrementOperand(MI, HII).getReg() == DepReg) { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 672 | return false; |
| 673 | } |
| 674 | |
Krzysztof Parzyszek | 8fb181c | 2016-08-01 17:55:48 +0000 | [diff] [blame] | 675 | if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() && |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 676 | getPostIncrementOperand(PacketMI, HII).getReg() == DepReg) { |
| 677 | // If source is post_inc, or absolute-set addressing, it can not feed |
| 678 | // into new value store |
| 679 | // r3 = memw(r2++#4) |
| 680 | // memw(r30 + #-1404) = r2.new -> can not be new value store |
| 681 | // arch spec section: 5.4.2.1. |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 682 | return false; |
| 683 | } |
| 684 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 685 | if (isLoadAbsSet(PacketMI) && getAbsSetOperand(PacketMI).getReg() == DepReg) |
| 686 | return false; |
| 687 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 688 | // If the source that feeds the store is predicated, new value store must |
Jyotsna Verma | 438cec5 | 2013-05-10 20:58:11 +0000 | [diff] [blame] | 689 | // also be predicated. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 690 | if (HII->isPredicated(PacketMI)) { |
| 691 | if (!HII->isPredicated(MI)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 692 | return false; |
| 693 | |
| 694 | // Check to make sure that they both will have their predicates |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 695 | // evaluate identically. |
Sirish Pande | 95d0117 | 2012-05-11 20:00:34 +0000 | [diff] [blame] | 696 | unsigned predRegNumSrc = 0; |
| 697 | unsigned predRegNumDst = 0; |
Craig Topper | 062a2ba | 2014-04-25 05:30:21 +0000 | [diff] [blame] | 698 | const TargetRegisterClass* predRegClass = nullptr; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 699 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 700 | // Get predicate register used in the source instruction. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 701 | for (auto &MO : PacketMI.operands()) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 702 | if (!MO.isReg()) |
| 703 | continue; |
| 704 | predRegNumSrc = MO.getReg(); |
| 705 | predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc); |
| 706 | if (predRegClass == &Hexagon::PredRegsRegClass) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 707 | break; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 708 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 709 | assert((predRegClass == &Hexagon::PredRegsRegClass) && |
| 710 | "predicate register not found in a predicated PacketMI instruction"); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 711 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 712 | // Get predicate register used in new-value store instruction. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 713 | for (auto &MO : MI.operands()) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 714 | if (!MO.isReg()) |
| 715 | continue; |
| 716 | predRegNumDst = MO.getReg(); |
| 717 | predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst); |
| 718 | if (predRegClass == &Hexagon::PredRegsRegClass) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 719 | break; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 720 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 721 | assert((predRegClass == &Hexagon::PredRegsRegClass) && |
| 722 | "predicate register not found in a predicated MI instruction"); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 723 | |
| 724 | // New-value register producer and user (store) need to satisfy these |
| 725 | // constraints: |
| 726 | // 1) Both instructions should be predicated on the same register. |
| 727 | // 2) If producer of the new-value register is .new predicated then store |
| 728 | // should also be .new predicated and if producer is not .new predicated |
| 729 | // then store should not be .new predicated. |
| 730 | // 3) Both new-value register producer and user should have same predicate |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 731 | // sense, i.e, either both should be negated or both should be non-negated. |
| 732 | if (predRegNumDst != predRegNumSrc || |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame] | 733 | HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) || |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 734 | getPredicateSense(MI, HII) != getPredicateSense(PacketMI, HII)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 735 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 736 | } |
| 737 | |
| 738 | // Make sure that other than the new-value register no other store instruction |
| 739 | // register has been modified in the same packet. Predicate registers can be |
| 740 | // modified by they should not be modified between the producer and the store |
| 741 | // instruction as it will make them both conditional on different values. |
| 742 | // We already know this to be true for all the instructions before and |
| 743 | // including PacketMI. Howerver, we need to perform the check for the |
| 744 | // remaining instructions in the packet. |
| 745 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 746 | unsigned StartCheck = 0; |
| 747 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 748 | for (auto I : CurrentPacketMIs) { |
| 749 | SUnit *TempSU = MIToSUnit.find(I)->second; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 750 | MachineInstr &TempMI = *TempSU->getInstr(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 751 | |
| 752 | // Following condition is true for all the instructions until PacketMI is |
| 753 | // reached (StartCheck is set to 0 before the for loop). |
| 754 | // StartCheck flag is 1 for all the instructions after PacketMI. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 755 | if (&TempMI != &PacketMI && !StartCheck) // Start processing only after |
| 756 | continue; // encountering PacketMI. |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 757 | |
| 758 | StartCheck = 1; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 759 | if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence. |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 760 | continue; |
| 761 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 762 | for (auto &MO : MI.operands()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 763 | if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 764 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 765 | } |
| 766 | |
Alp Toker | f907b89 | 2013-12-05 05:44:44 +0000 | [diff] [blame] | 767 | // Make sure that for non-POST_INC stores: |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 768 | // 1. The only use of reg is DepReg and no other registers. |
| 769 | // This handles V4 base+index registers. |
| 770 | // The following store can not be dot new. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 771 | // Eg. r0 = add(r0, #3) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 772 | // memw(r1+r0<<#2) = r0 |
Krzysztof Parzyszek | 8fb181c | 2016-08-01 17:55:48 +0000 | [diff] [blame] | 773 | if (!HII->isPostIncrement(MI)) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 774 | for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) { |
| 775 | const MachineOperand &MO = MI.getOperand(opNum); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 776 | if (MO.isReg() && MO.getReg() == DepReg) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 777 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 778 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 779 | } |
| 780 | |
| 781 | // If data definition is because of implicit definition of the register, |
| 782 | // do not newify the store. Eg. |
Francis Visoiu Mistrih | a8a83d1 | 2017-12-07 10:40:31 +0000 | [diff] [blame] | 783 | // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12 |
| 784 | // S2_storerh_io %r8, 2, killed %r12; mem:ST2[%scevgep343] |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 785 | for (auto &MO : PacketMI.operands()) { |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 786 | if (MO.isRegMask() && MO.clobbersPhysReg(DepReg)) |
| 787 | return false; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 788 | if (!MO.isReg() || !MO.isDef() || !MO.isImplicit()) |
| 789 | continue; |
| 790 | unsigned R = MO.getReg(); |
| 791 | if (R == DepReg || HRI->isSuperRegister(DepReg, R)) |
| 792 | return false; |
| 793 | } |
| 794 | |
| 795 | // Handle imp-use of super reg case. There is a target independent side |
| 796 | // change that should prevent this situation but I am handling it for |
| 797 | // just-in-case. For example, we cannot newify R2 in the following case: |
Francis Visoiu Mistrih | a8a83d1 | 2017-12-07 10:40:31 +0000 | [diff] [blame] | 798 | // %r3 = A2_tfrsi 0; |
| 799 | // S2_storeri_io killed %r0, 0, killed %r2, implicit killed %d1; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 800 | for (auto &MO : MI.operands()) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 801 | if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg) |
| 802 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 803 | } |
| 804 | |
| 805 | // Can be dot new store. |
| 806 | return true; |
| 807 | } |
| 808 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 809 | // Can this MI to promoted to either new value store or new value jump. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 810 | bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 811 | const SUnit *PacketSU, unsigned DepReg, |
| 812 | MachineBasicBlock::iterator &MII) { |
| 813 | if (!HII->mayBeNewStore(MI)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 814 | return false; |
| 815 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 816 | // Check to see the store can be new value'ed. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 817 | MachineInstr &PacketMI = *PacketSU->getInstr(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 818 | if (canPromoteToNewValueStore(MI, PacketMI, DepReg)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 819 | return true; |
| 820 | |
| 821 | // Check to see the compare/jump can be new value'ed. |
| 822 | // This is done as a pass on its own. Don't need to check it here. |
| 823 | return false; |
| 824 | } |
| 825 | |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 826 | static bool isImplicitDependency(const MachineInstr &I, bool CheckDef, |
| 827 | unsigned DepReg) { |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 828 | for (auto &MO : I.operands()) { |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 829 | if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(DepReg)) |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 830 | return true; |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 831 | if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit()) |
| 832 | continue; |
| 833 | if (CheckDef == MO.isDef()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 834 | return true; |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 835 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 836 | return false; |
| 837 | } |
| 838 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 839 | // Check to see if an instruction can be dot new |
| 840 | // There are three kinds. |
| 841 | // 1. dot new on predicate - V2/V3/V4 |
| 842 | // 2. dot new on stores NV/ST - V4 |
| 843 | // 3. dot new on jump NV/J - V4 -- This is generated in a pass. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 844 | bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 845 | const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, |
| 846 | const TargetRegisterClass* RC) { |
Jyotsna Verma | a46059b | 2013-03-28 19:44:04 +0000 | [diff] [blame] | 847 | // Already a dot new instruction. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 848 | if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 849 | return false; |
| 850 | |
Krzysztof Parzyszek | 2a48059 | 2016-07-26 20:30:30 +0000 | [diff] [blame] | 851 | if (!isNewifiable(MI, RC)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 852 | return false; |
| 853 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 854 | const MachineInstr &PI = *PacketSU->getInstr(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 855 | |
| 856 | // The "new value" cannot come from inline asm. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 857 | if (PI.isInlineAsm()) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 858 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 859 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 860 | // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no |
| 861 | // sense. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 862 | if (PI.isImplicitDef()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 863 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 864 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 865 | // If dependency is trough an implicitly defined register, we should not |
| 866 | // newify the use. |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 867 | if (isImplicitDependency(PI, true, DepReg) || |
| 868 | isImplicitDependency(MI, false, DepReg)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 869 | return false; |
| 870 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 871 | const MCInstrDesc& MCID = PI.getDesc(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 872 | const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF); |
Krzysztof Parzyszek | 5577297 | 2017-09-15 15:46:05 +0000 | [diff] [blame] | 873 | if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 874 | return false; |
| 875 | |
| 876 | // predicate .new |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 877 | if (RC == &Hexagon::PredRegsRegClass) |
Krzysztof Parzyszek | 3cf1657 | 2017-06-01 18:02:40 +0000 | [diff] [blame] | 878 | return HII->predCanBeUsedAsDotNew(PI, DepReg); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 879 | |
| 880 | if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI)) |
| 881 | return false; |
| 882 | |
| 883 | // Create a dot new machine instruction to see if resources can be |
| 884 | // allocated. If not, bail out now. |
| 885 | int NewOpcode = HII->getDotNewOp(MI); |
| 886 | const MCInstrDesc &D = HII->get(NewOpcode); |
| 887 | MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc()); |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 888 | bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 889 | MF.DeleteMachineInstr(NewMI); |
| 890 | if (!ResourcesAvailable) |
| 891 | return false; |
| 892 | |
| 893 | // New Value Store only. New Value Jump generated as a separate pass. |
| 894 | if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII)) |
| 895 | return false; |
| 896 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 897 | return true; |
| 898 | } |
| 899 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 900 | // Go through the packet instructions and search for an anti dependency between |
| 901 | // them and DepReg from MI. Consider this case: |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 902 | // Trying to add |
Francis Visoiu Mistrih | a8a83d1 | 2017-12-07 10:40:31 +0000 | [diff] [blame] | 903 | // a) %r1 = TFRI_cdNotPt %p3, 2 |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 904 | // to this packet: |
| 905 | // { |
Francis Visoiu Mistrih | a8a83d1 | 2017-12-07 10:40:31 +0000 | [diff] [blame] | 906 | // b) %p0 = C2_or killed %p3, killed %p0 |
| 907 | // c) %p3 = C2_tfrrp %r23 |
| 908 | // d) %r1 = C2_cmovenewit %p3, 4 |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 909 | // } |
| 910 | // The P3 from a) and d) will be complements after |
| 911 | // a)'s P3 is converted to .new form |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 912 | // Anti-dep between c) and b) is irrelevant for this case |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 913 | bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 914 | unsigned DepReg) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 915 | SUnit *PacketSUDep = MIToSUnit.find(&MI)->second; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 916 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 917 | for (auto I : CurrentPacketMIs) { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 918 | // We only care for dependencies to predicated instructions |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame] | 919 | if (!HII->isPredicated(*I)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 920 | continue; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 921 | |
| 922 | // Scheduling Unit for current insn in the packet |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 923 | SUnit *PacketSU = MIToSUnit.find(I)->second; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 924 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 925 | // Look at dependencies between current members of the packet and |
| 926 | // predicate defining instruction MI. Make sure that dependency is |
| 927 | // on the exact register we care about. |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 928 | if (PacketSU->isSucc(PacketSUDep)) { |
| 929 | for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 930 | auto &Dep = PacketSU->Succs[i]; |
| 931 | if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti && |
| 932 | Dep.getReg() == DepReg) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 933 | return true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 934 | } |
| 935 | } |
| 936 | } |
| 937 | |
| 938 | return false; |
| 939 | } |
| 940 | |
Jyotsna Verma | 11bd54a | 2013-05-14 16:36:34 +0000 | [diff] [blame] | 941 | /// Gets the predicate register of a predicated instruction. |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame] | 942 | static unsigned getPredicatedRegister(MachineInstr &MI, |
Benjamin Kramer | e79beac | 2013-05-23 15:43:11 +0000 | [diff] [blame] | 943 | const HexagonInstrInfo *QII) { |
Jyotsna Verma | 11bd54a | 2013-05-14 16:36:34 +0000 | [diff] [blame] | 944 | /// We use the following rule: The first predicate register that is a use is |
| 945 | /// the predicate register of a predicated instruction. |
Jyotsna Verma | 11bd54a | 2013-05-14 16:36:34 +0000 | [diff] [blame] | 946 | assert(QII->isPredicated(MI) && "Must be predicated instruction"); |
| 947 | |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame] | 948 | for (auto &Op : MI.operands()) { |
Jyotsna Verma | 11bd54a | 2013-05-14 16:36:34 +0000 | [diff] [blame] | 949 | if (Op.isReg() && Op.getReg() && Op.isUse() && |
| 950 | Hexagon::PredRegsRegClass.contains(Op.getReg())) |
| 951 | return Op.getReg(); |
| 952 | } |
| 953 | |
| 954 | llvm_unreachable("Unknown instruction operand layout"); |
Jyotsna Verma | 11bd54a | 2013-05-14 16:36:34 +0000 | [diff] [blame] | 955 | return 0; |
| 956 | } |
| 957 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 958 | // Given two predicated instructions, this function detects whether |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 959 | // the predicates are complements. |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame] | 960 | bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1, |
| 961 | MachineInstr &MI2) { |
Jyotsna Verma | 11bd54a | 2013-05-14 16:36:34 +0000 | [diff] [blame] | 962 | // If we don't know the predicate sense of the instructions bail out early, we |
| 963 | // need it later. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 964 | if (getPredicateSense(MI1, HII) == PK_Unknown || |
| 965 | getPredicateSense(MI2, HII) == PK_Unknown) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 966 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 967 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 968 | // Scheduling unit for candidate. |
Duncan P. N. Exon Smith | 6307eb5 | 2016-02-23 02:46:52 +0000 | [diff] [blame] | 969 | SUnit *SU = MIToSUnit[&MI1]; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 970 | |
| 971 | // One corner case deals with the following scenario: |
| 972 | // Trying to add |
Francis Visoiu Mistrih | a8a83d1 | 2017-12-07 10:40:31 +0000 | [diff] [blame] | 973 | // a) %r24 = A2_tfrt %p0, %r25 |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 974 | // to this packet: |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 975 | // { |
Francis Visoiu Mistrih | a8a83d1 | 2017-12-07 10:40:31 +0000 | [diff] [blame] | 976 | // b) %r25 = A2_tfrf %p0, %r24 |
| 977 | // c) %p0 = C2_cmpeqi %r26, 1 |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 978 | // } |
| 979 | // |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 980 | // On general check a) and b) are complements, but presence of c) will |
| 981 | // convert a) to .new form, and then it is not a complement. |
| 982 | // We attempt to detect it by analyzing existing dependencies in the packet. |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 983 | |
| 984 | // Analyze relationships between all existing members of the packet. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 985 | // Look for Anti dependecy on the same predicate reg as used in the |
| 986 | // candidate. |
| 987 | for (auto I : CurrentPacketMIs) { |
| 988 | // Scheduling Unit for current insn in the packet. |
| 989 | SUnit *PacketSU = MIToSUnit.find(I)->second; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 990 | |
| 991 | // If this instruction in the packet is succeeded by the candidate... |
| 992 | if (PacketSU->isSucc(SU)) { |
| 993 | for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 994 | auto Dep = PacketSU->Succs[i]; |
| 995 | // The corner case exist when there is true data dependency between |
| 996 | // candidate and one of current packet members, this dep is on |
| 997 | // predicate reg, and there already exist anti dep on the same pred in |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 998 | // the packet. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 999 | if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data && |
| 1000 | Hexagon::PredRegsRegClass.contains(Dep.getReg())) { |
| 1001 | // Here I know that I is predicate setting instruction with true |
| 1002 | // data dep to candidate on the register we care about - c) in the |
| 1003 | // above example. Now I need to see if there is an anti dependency |
| 1004 | // from c) to any other instruction in the same packet on the pred |
| 1005 | // reg of interest. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1006 | if (restrictingDepExistInPacket(*I, Dep.getReg())) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1007 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1008 | } |
| 1009 | } |
| 1010 | } |
| 1011 | } |
| 1012 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1013 | // If the above case does not apply, check regular complement condition. |
| 1014 | // Check that the predicate register is the same and that the predicate |
| 1015 | // sense is different We also need to differentiate .old vs. .new: !p0 |
| 1016 | // is not complementary to p0.new. |
| 1017 | unsigned PReg1 = getPredicatedRegister(MI1, HII); |
| 1018 | unsigned PReg2 = getPredicatedRegister(MI2, HII); |
| 1019 | return PReg1 == PReg2 && |
| 1020 | Hexagon::PredRegsRegClass.contains(PReg1) && |
| 1021 | Hexagon::PredRegsRegClass.contains(PReg2) && |
| 1022 | getPredicateSense(MI1, HII) != getPredicateSense(MI2, HII) && |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1023 | HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1024 | } |
| 1025 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1026 | // Initialize packetizer flags. |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1027 | void HexagonPacketizerList::initPacketizerState() { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1028 | Dependence = false; |
| 1029 | PromotedToDotNew = false; |
| 1030 | GlueToNewValueJump = false; |
| 1031 | GlueAllocframeStore = false; |
| 1032 | FoundSequentialDependence = false; |
Krzysztof Parzyszek | 8f174dd | 2017-10-11 15:51:44 +0000 | [diff] [blame] | 1033 | ChangedOffset = INT64_MAX; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1034 | } |
| 1035 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1036 | // Ignore bundling of pseudo instructions. |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1037 | bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI, |
| 1038 | const MachineBasicBlock *) { |
Shiva Chen | 801bf7e | 2018-05-09 02:42:00 +0000 | [diff] [blame] | 1039 | if (MI.isDebugInstr()) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1040 | return true; |
| 1041 | |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1042 | if (MI.isCFIInstruction()) |
Krzysztof Parzyszek | 6bbcb31 | 2015-04-22 15:47:35 +0000 | [diff] [blame] | 1043 | return false; |
| 1044 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1045 | // We must print out inline assembly. |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1046 | if (MI.isInlineAsm()) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1047 | return false; |
| 1048 | |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1049 | if (MI.isImplicitDef()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1050 | return false; |
| 1051 | |
| 1052 | // We check if MI has any functional units mapped to it. If it doesn't, |
| 1053 | // we ignore the instruction. |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1054 | const MCInstrDesc& TID = MI.getDesc(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1055 | auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass()); |
Hal Finkel | 8db5547 | 2012-06-22 20:27:13 +0000 | [diff] [blame] | 1056 | unsigned FuncUnits = IS->getUnits(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1057 | return !FuncUnits; |
| 1058 | } |
| 1059 | |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1060 | bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) { |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1061 | // Ensure any bundles created by gather packetize remain seperate. |
| 1062 | if (MI.isBundle()) |
| 1063 | return true; |
| 1064 | |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1065 | if (MI.isEHLabel() || MI.isCFIInstruction()) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1066 | return true; |
| 1067 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1068 | // Consider inline asm to not be a solo instruction by default. |
| 1069 | // Inline asm will be put in a packet temporarily, but then it will be |
| 1070 | // removed, and placed outside of the packet (before or after, depending |
| 1071 | // on dependencies). This is to reduce the impact of inline asm as a |
| 1072 | // "packet splitting" instruction. |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1073 | if (MI.isInlineAsm() && !ScheduleInlineAsm) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1074 | return true; |
| 1075 | |
| 1076 | // From Hexagon V4 Programmer's Reference Manual 3.4.4 Grouping constraints: |
| 1077 | // trap, pause, barrier, icinva, isync, and syncht are solo instructions. |
| 1078 | // They must not be grouped with other instructions in a packet. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1079 | if (isSchedBarrier(MI)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1080 | return true; |
| 1081 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1082 | if (HII->isSolo(MI)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1083 | return true; |
| 1084 | |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1085 | if (MI.getOpcode() == Hexagon::A2_nop) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1086 | return true; |
| 1087 | |
| 1088 | return false; |
| 1089 | } |
| 1090 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1091 | // Quick check if instructions MI and MJ cannot coexist in the same packet. |
| 1092 | // Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm", |
| 1093 | // but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm". |
| 1094 | // For full test call this function twice: |
| 1095 | // cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI) |
| 1096 | // Doing the test only one way saves the amount of code in this function, |
| 1097 | // since every test would need to be repeated with the MI and MJ reversed. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1098 | static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ, |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1099 | const HexagonInstrInfo &HII) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1100 | const MachineFunction *MF = MI.getParent()->getParent(); |
Krzysztof Parzyszek | d8b780d | 2018-06-20 13:56:09 +0000 | [diff] [blame] | 1101 | if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() && |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1102 | HII.isHVXMemWithAIndirect(MI, MJ)) |
| 1103 | return true; |
| 1104 | |
| 1105 | // An inline asm cannot be together with a branch, because we may not be |
| 1106 | // able to remove the asm out after packetizing (i.e. if the asm must be |
| 1107 | // moved past the bundle). Similarly, two asms cannot be together to avoid |
| 1108 | // complications when determining their relative order outside of a bundle. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1109 | if (MI.isInlineAsm()) |
| 1110 | return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() || |
| 1111 | MJ.isCall() || MJ.isTerminator(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1112 | |
Krzysztof Parzyszek | 639545b | 2016-08-19 16:57:05 +0000 | [diff] [blame] | 1113 | switch (MI.getOpcode()) { |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1114 | case Hexagon::S2_storew_locked: |
| 1115 | case Hexagon::S4_stored_locked: |
| 1116 | case Hexagon::L2_loadw_locked: |
| 1117 | case Hexagon::L4_loadd_locked: |
Krzysztof Parzyszek | 5c2944c | 2018-06-19 17:26:20 +0000 | [diff] [blame] | 1118 | case Hexagon::Y2_dccleana: |
| 1119 | case Hexagon::Y2_dccleaninva: |
| 1120 | case Hexagon::Y2_dcinva: |
| 1121 | case Hexagon::Y2_dczeroa: |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1122 | case Hexagon::Y4_l2fetch: |
| 1123 | case Hexagon::Y5_l2fetch: { |
Krzysztof Parzyszek | 639545b | 2016-08-19 16:57:05 +0000 | [diff] [blame] | 1124 | // These instructions can only be grouped with ALU32 or non-floating-point |
| 1125 | // XTYPE instructions. Since there is no convenient way of identifying fp |
| 1126 | // XTYPE instructions, only allow grouping with ALU32 for now. |
| 1127 | unsigned TJ = HII.getType(MJ); |
Krzysztof Parzyszek | 5ea971c | 2017-02-07 17:47:37 +0000 | [diff] [blame] | 1128 | if (TJ != HexagonII::TypeALU32_2op && |
| 1129 | TJ != HexagonII::TypeALU32_3op && |
| 1130 | TJ != HexagonII::TypeALU32_ADDI) |
Krzysztof Parzyszek | 639545b | 2016-08-19 16:57:05 +0000 | [diff] [blame] | 1131 | return true; |
| 1132 | break; |
| 1133 | } |
| 1134 | default: |
| 1135 | break; |
| 1136 | } |
| 1137 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1138 | // "False" really means that the quick check failed to determine if |
| 1139 | // I and J cannot coexist. |
| 1140 | return false; |
| 1141 | } |
| 1142 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1143 | // Full, symmetric check. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1144 | bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI, |
| 1145 | const MachineInstr &MJ) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1146 | return cannotCoexistAsymm(MI, MJ, *HII) || cannotCoexistAsymm(MJ, MI, *HII); |
| 1147 | } |
| 1148 | |
| 1149 | void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) { |
| 1150 | for (auto &B : MF) { |
| 1151 | MachineBasicBlock::iterator BundleIt; |
| 1152 | MachineBasicBlock::instr_iterator NextI; |
| 1153 | for (auto I = B.instr_begin(), E = B.instr_end(); I != E; I = NextI) { |
| 1154 | NextI = std::next(I); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1155 | MachineInstr &MI = *I; |
| 1156 | if (MI.isBundle()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1157 | BundleIt = I; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1158 | if (!MI.isInsideBundle()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1159 | continue; |
| 1160 | |
| 1161 | // Decide on where to insert the instruction that we are pulling out. |
| 1162 | // Debug instructions always go before the bundle, but the placement of |
| 1163 | // INLINE_ASM depends on potential dependencies. By default, try to |
| 1164 | // put it before the bundle, but if the asm writes to a register that |
| 1165 | // other instructions in the bundle read, then we need to place it |
| 1166 | // after the bundle (to preserve the bundle semantics). |
| 1167 | bool InsertBeforeBundle; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1168 | if (MI.isInlineAsm()) |
| 1169 | InsertBeforeBundle = !hasWriteToReadDep(MI, *BundleIt, HRI); |
| 1170 | else if (MI.isDebugValue()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1171 | InsertBeforeBundle = true; |
| 1172 | else |
| 1173 | continue; |
| 1174 | |
| 1175 | BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle); |
| 1176 | } |
| 1177 | } |
| 1178 | } |
| 1179 | |
| 1180 | // Check if a given instruction is of class "system". |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1181 | static bool isSystemInstr(const MachineInstr &MI) { |
| 1182 | unsigned Opc = MI.getOpcode(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1183 | switch (Opc) { |
| 1184 | case Hexagon::Y2_barrier: |
| 1185 | case Hexagon::Y2_dcfetchbo: |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1186 | case Hexagon::Y4_l2fetch: |
| 1187 | case Hexagon::Y5_l2fetch: |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1188 | return true; |
| 1189 | } |
| 1190 | return false; |
| 1191 | } |
| 1192 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1193 | bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I, |
| 1194 | const MachineInstr &J) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1195 | // The dependence graph may not include edges between dead definitions, |
| 1196 | // so without extra checks, we could end up packetizing two instruction |
| 1197 | // defining the same (dead) register. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1198 | if (I.isCall() || J.isCall()) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1199 | return false; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1200 | if (HII->isPredicated(I) || HII->isPredicated(J)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1201 | return false; |
| 1202 | |
| 1203 | BitVector DeadDefs(Hexagon::NUM_TARGET_REGS); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1204 | for (auto &MO : I.operands()) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1205 | if (!MO.isReg() || !MO.isDef() || !MO.isDead()) |
| 1206 | continue; |
| 1207 | DeadDefs[MO.getReg()] = true; |
| 1208 | } |
| 1209 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1210 | for (auto &MO : J.operands()) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1211 | if (!MO.isReg() || !MO.isDef() || !MO.isDead()) |
| 1212 | continue; |
| 1213 | unsigned R = MO.getReg(); |
| 1214 | if (R != Hexagon::USR_OVF && DeadDefs[R]) |
| 1215 | return true; |
| 1216 | } |
| 1217 | return false; |
| 1218 | } |
| 1219 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1220 | bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I, |
| 1221 | const MachineInstr &J) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1222 | // A save callee-save register function call can only be in a packet |
| 1223 | // with instructions that don't write to the callee-save registers. |
| 1224 | if ((HII->isSaveCalleeSavedRegsCall(I) && |
| 1225 | doesModifyCalleeSavedReg(J, HRI)) || |
| 1226 | (HII->isSaveCalleeSavedRegsCall(J) && |
| 1227 | doesModifyCalleeSavedReg(I, HRI))) |
| 1228 | return true; |
| 1229 | |
| 1230 | // Two control flow instructions cannot go in the same packet. |
| 1231 | if (isControlFlow(I) && isControlFlow(J)) |
| 1232 | return true; |
| 1233 | |
| 1234 | // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot |
| 1235 | // contain a speculative indirect jump, |
| 1236 | // a new-value compare jump or a dealloc_return. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1237 | auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool { |
| 1238 | if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1239 | return true; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1240 | if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1241 | return true; |
| 1242 | return false; |
| 1243 | }; |
| 1244 | |
| 1245 | if (HII->isLoopN(I) && isBadForLoopN(J)) |
| 1246 | return true; |
| 1247 | if (HII->isLoopN(J) && isBadForLoopN(I)) |
| 1248 | return true; |
| 1249 | |
| 1250 | // dealloc_return cannot appear in the same packet as a conditional or |
| 1251 | // unconditional jump. |
| 1252 | return HII->isDeallocRet(I) && |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1253 | (J.isBranch() || J.isCall() || J.isBarrier()); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1254 | } |
| 1255 | |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 1256 | bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I, |
| 1257 | const MachineInstr &J) { |
| 1258 | // Adding I to a packet that has J. |
| 1259 | |
| 1260 | // Regmasks are not reflected in the scheduling dependency graph, so |
| 1261 | // we need to check them manually. This code assumes that regmasks only |
| 1262 | // occur on calls, and the problematic case is when we add an instruction |
| 1263 | // defining a register R to a packet that has a call that clobbers R via |
| 1264 | // a regmask. Those cannot be packetized together, because the call will |
| 1265 | // be executed last. That's also a reson why it is ok to add a call |
| 1266 | // clobbering R to a packet that defines R. |
| 1267 | |
| 1268 | // Look for regmasks in J. |
| 1269 | for (const MachineOperand &OpJ : J.operands()) { |
| 1270 | if (!OpJ.isRegMask()) |
| 1271 | continue; |
| 1272 | assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call"); |
| 1273 | for (const MachineOperand &OpI : I.operands()) { |
| 1274 | if (OpI.isReg()) { |
| 1275 | if (OpJ.clobbersPhysReg(OpI.getReg())) |
| 1276 | return true; |
| 1277 | } else if (OpI.isRegMask()) { |
| 1278 | // Both are regmasks. Assume that they intersect. |
| 1279 | return true; |
| 1280 | } |
| 1281 | } |
| 1282 | } |
| 1283 | return false; |
| 1284 | } |
| 1285 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1286 | bool HexagonPacketizerList::hasV4SpecificDependence(const MachineInstr &I, |
| 1287 | const MachineInstr &J) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1288 | bool SysI = isSystemInstr(I), SysJ = isSystemInstr(J); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1289 | bool StoreI = I.mayStore(), StoreJ = J.mayStore(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1290 | if ((SysI && StoreJ) || (SysJ && StoreI)) |
| 1291 | return true; |
| 1292 | |
| 1293 | if (StoreI && StoreJ) { |
| 1294 | if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I)) |
| 1295 | return true; |
| 1296 | } else { |
| 1297 | // A memop cannot be in the same packet with another memop or a store. |
| 1298 | // Two stores can be together, but here I and J cannot both be stores. |
| 1299 | bool MopStI = HII->isMemOp(I) || StoreI; |
| 1300 | bool MopStJ = HII->isMemOp(J) || StoreJ; |
| 1301 | if (MopStI && MopStJ) |
| 1302 | return true; |
| 1303 | } |
| 1304 | |
| 1305 | return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J)); |
| 1306 | } |
| 1307 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1308 | // SUI is the current instruction that is out side of the current packet. |
| 1309 | // SUJ is the current instruction inside the current packet against which that |
| 1310 | // SUI will be packetized. |
| 1311 | bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1312 | assert(SUI->getInstr() && SUJ->getInstr()); |
| 1313 | MachineInstr &I = *SUI->getInstr(); |
| 1314 | MachineInstr &J = *SUJ->getInstr(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1315 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1316 | // Clear IgnoreDepMIs when Packet starts. |
| 1317 | if (CurrentPacketMIs.size() == 1) |
| 1318 | IgnoreDepMIs.clear(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1319 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1320 | MachineBasicBlock::iterator II = I.getIterator(); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1321 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1322 | // Solo instructions cannot go in the packet. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1323 | assert(!isSoloInstruction(I) && "Unexpected solo instr!"); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1324 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1325 | if (cannotCoexist(I, J)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1326 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1327 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1328 | Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J); |
| 1329 | if (Dependence) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1330 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1331 | |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 1332 | // Regmasks are not accounted for in the scheduling graph, so we need |
| 1333 | // to explicitly check for dependencies caused by them. They should only |
| 1334 | // appear on calls, so it's not too pessimistic to reject all regmask |
| 1335 | // dependencies. |
| 1336 | Dependence = hasRegMaskDependence(I, J); |
| 1337 | if (Dependence) |
| 1338 | return false; |
| 1339 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1340 | // V4 allows dual stores. It does not allow second store, if the first |
| 1341 | // store is not in SLOT0. New value store, new value jump, dealloc_return |
| 1342 | // and memop always take SLOT0. Arch spec 3.4.4.2. |
| 1343 | Dependence = hasV4SpecificDependence(I, J); |
| 1344 | if (Dependence) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1345 | return false; |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1346 | |
| 1347 | // If an instruction feeds new value jump, glue it. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1348 | MachineBasicBlock::iterator NextMII = I.getIterator(); |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1349 | ++NextMII; |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1350 | if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) { |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 1351 | MachineInstr &NextMI = *NextMII; |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1352 | |
| 1353 | bool secondRegMatch = false; |
Duncan P. N. Exon Smith | 98226e3 | 2016-07-12 01:55:32 +0000 | [diff] [blame] | 1354 | const MachineOperand &NOp0 = NextMI.getOperand(0); |
| 1355 | const MachineOperand &NOp1 = NextMI.getOperand(1); |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1356 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1357 | if (NOp1.isReg() && I.getOperand(0).getReg() == NOp1.getReg()) |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1358 | secondRegMatch = true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1359 | |
Krzysztof Parzyszek | c4a9a8d | 2017-10-11 21:20:43 +0000 | [diff] [blame] | 1360 | for (MachineInstr *PI : CurrentPacketMIs) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1361 | // NVJ can not be part of the dual jump - Arch Spec: section 7.8. |
Krzysztof Parzyszek | c4a9a8d | 2017-10-11 21:20:43 +0000 | [diff] [blame] | 1362 | if (PI->isCall()) { |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1363 | Dependence = true; |
| 1364 | break; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1365 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1366 | // Validate: |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1367 | // 1. Packet does not have a store in it. |
| 1368 | // 2. If the first operand of the nvj is newified, and the second |
| 1369 | // operand is also a reg, it (second reg) is not defined in |
| 1370 | // the same packet. |
| 1371 | // 3. If the second operand of the nvj is newified, (which means |
| 1372 | // first operand is also a reg), first reg is not defined in |
| 1373 | // the same packet. |
Krzysztof Parzyszek | c4a9a8d | 2017-10-11 21:20:43 +0000 | [diff] [blame] | 1374 | if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() || |
| 1375 | HII->isLoopN(*PI)) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1376 | Dependence = true; |
| 1377 | break; |
| 1378 | } |
| 1379 | // Check #2/#3. |
| 1380 | const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1; |
Krzysztof Parzyszek | c4a9a8d | 2017-10-11 21:20:43 +0000 | [diff] [blame] | 1381 | if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) { |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1382 | Dependence = true; |
| 1383 | break; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1384 | } |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1385 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1386 | |
Krzysztof Parzyszek | c4a9a8d | 2017-10-11 21:20:43 +0000 | [diff] [blame] | 1387 | GlueToNewValueJump = true; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1388 | if (Dependence) |
Colin LeMahieu | 4fd203d | 2015-02-09 21:56:37 +0000 | [diff] [blame] | 1389 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1390 | } |
| 1391 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1392 | // There no dependency between a prolog instruction and its successor. |
| 1393 | if (!SUJ->isSucc(SUI)) |
| 1394 | return true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1395 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1396 | for (unsigned i = 0; i < SUJ->Succs.size(); ++i) { |
| 1397 | if (FoundSequentialDependence) |
| 1398 | break; |
| 1399 | |
| 1400 | if (SUJ->Succs[i].getSUnit() != SUI) |
| 1401 | continue; |
| 1402 | |
| 1403 | SDep::Kind DepType = SUJ->Succs[i].getKind(); |
| 1404 | // For direct calls: |
| 1405 | // Ignore register dependences for call instructions for packetization |
| 1406 | // purposes except for those due to r31 and predicate registers. |
| 1407 | // |
| 1408 | // For indirect calls: |
| 1409 | // Same as direct calls + check for true dependences to the register |
| 1410 | // used in the indirect call. |
| 1411 | // |
| 1412 | // We completely ignore Order dependences for call instructions. |
| 1413 | // |
| 1414 | // For returns: |
| 1415 | // Ignore register dependences for return instructions like jumpr, |
| 1416 | // dealloc return unless we have dependencies on the explicit uses |
| 1417 | // of the registers used by jumpr (like r31) or dealloc return |
| 1418 | // (like r29 or r30). |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1419 | unsigned DepReg = 0; |
| 1420 | const TargetRegisterClass *RC = nullptr; |
| 1421 | if (DepType == SDep::Data) { |
| 1422 | DepReg = SUJ->Succs[i].getReg(); |
| 1423 | RC = HRI->getMinimalPhysRegClass(DepReg); |
| 1424 | } |
| 1425 | |
Krzysztof Parzyszek | 38e2ccc | 2016-08-23 16:01:01 +0000 | [diff] [blame] | 1426 | if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1427 | if (!isRegDependence(DepType)) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1428 | continue; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1429 | if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg())) |
| 1430 | continue; |
| 1431 | } |
| 1432 | |
| 1433 | if (DepType == SDep::Data) { |
| 1434 | if (canPromoteToDotCur(J, SUJ, DepReg, II, RC)) |
| 1435 | if (promoteToDotCur(J, DepType, II, RC)) |
| 1436 | continue; |
| 1437 | } |
| 1438 | |
| 1439 | // Data dpendence ok if we have load.cur. |
| 1440 | if (DepType == SDep::Data && HII->isDotCurInst(J)) { |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1441 | if (HII->isHVXVec(I)) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1442 | continue; |
| 1443 | } |
| 1444 | |
| 1445 | // For instructions that can be promoted to dot-new, try to promote. |
| 1446 | if (DepType == SDep::Data) { |
| 1447 | if (canPromoteToDotNew(I, SUJ, DepReg, II, RC)) { |
| 1448 | if (promoteToDotNew(I, DepType, II, RC)) { |
| 1449 | PromotedToDotNew = true; |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1450 | if (cannotCoexist(I, J)) |
| 1451 | FoundSequentialDependence = true; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1452 | continue; |
| 1453 | } |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1454 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1455 | if (HII->isNewValueJump(I)) |
| 1456 | continue; |
| 1457 | } |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1458 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1459 | // For predicated instructions, if the predicates are complements then |
| 1460 | // there can be no dependence. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1461 | if (HII->isPredicated(I) && HII->isPredicated(J) && |
| 1462 | arePredicatesComplements(I, J)) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1463 | // Not always safe to do this translation. |
| 1464 | // DAG Builder attempts to reduce dependence edges using transitive |
| 1465 | // nature of dependencies. Here is an example: |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1466 | // |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1467 | // r0 = tfr_pt ... (1) |
| 1468 | // r0 = tfr_pf ... (2) |
| 1469 | // r0 = tfr_pt ... (3) |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1470 | // |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1471 | // There will be an output dependence between (1)->(2) and (2)->(3). |
| 1472 | // However, there is no dependence edge between (1)->(3). This results |
| 1473 | // in all 3 instructions going in the same packet. We ignore dependce |
| 1474 | // only once to avoid this situation. |
David Majnemer | 0d955d0 | 2016-08-11 22:21:41 +0000 | [diff] [blame] | 1475 | auto Itr = find(IgnoreDepMIs, &J); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1476 | if (Itr != IgnoreDepMIs.end()) { |
| 1477 | Dependence = true; |
| 1478 | return false; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1479 | } |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1480 | IgnoreDepMIs.push_back(&I); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1481 | continue; |
| 1482 | } |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1483 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1484 | // Ignore Order dependences between unconditional direct branches |
| 1485 | // and non-control-flow instructions. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1486 | if (isDirectJump(I) && !J.isBranch() && !J.isCall() && |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1487 | DepType == SDep::Order) |
| 1488 | continue; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1489 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1490 | // Ignore all dependences for jumps except for true and output |
| 1491 | // dependences. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1492 | if (I.isConditionalBranch() && DepType != SDep::Data && |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1493 | DepType != SDep::Output) |
| 1494 | continue; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1495 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1496 | if (DepType == SDep::Output) { |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1497 | FoundSequentialDependence = true; |
| 1498 | break; |
| 1499 | } |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1500 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1501 | // For Order dependences: |
| 1502 | // 1. On V4 or later, volatile loads/stores can be packetized together, |
| 1503 | // unless other rules prevent is. |
| 1504 | // 2. Store followed by a load is not allowed. |
| 1505 | // 3. Store followed by a store is only valid on V4 or later. |
| 1506 | // 4. Load followed by any memory operation is allowed. |
| 1507 | if (DepType == SDep::Order) { |
| 1508 | if (!PacketizeVolatiles) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1509 | bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1510 | if (OrdRefs) { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1511 | FoundSequentialDependence = true; |
| 1512 | break; |
| 1513 | } |
| 1514 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1515 | // J is first, I is second. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1516 | bool LoadJ = J.mayLoad(), StoreJ = J.mayStore(); |
| 1517 | bool LoadI = I.mayLoad(), StoreI = I.mayStore(); |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1518 | bool NVStoreJ = HII->isNewValueStore(J); |
| 1519 | bool NVStoreI = HII->isNewValueStore(I); |
| 1520 | bool IsVecJ = HII->isHVXVec(J); |
| 1521 | bool IsVecI = HII->isHVXVec(I); |
| 1522 | |
Krzysztof Parzyszek | d8b780d | 2018-06-20 13:56:09 +0000 | [diff] [blame] | 1523 | if (Slot1Store && MF.getSubtarget<HexagonSubtarget>().hasV65Ops() && |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1524 | ((LoadJ && StoreI && !NVStoreI) || |
| 1525 | (StoreJ && LoadI && !NVStoreJ)) && |
| 1526 | (J.getOpcode() != Hexagon::S2_allocframe && |
| 1527 | I.getOpcode() != Hexagon::S2_allocframe) && |
| 1528 | (J.getOpcode() != Hexagon::L2_deallocframe && |
| 1529 | I.getOpcode() != Hexagon::L2_deallocframe) && |
| 1530 | (!HII->isMemOp(J) && !HII->isMemOp(I)) && (!IsVecJ && !IsVecI)) |
| 1531 | setmemShufDisabled(true); |
| 1532 | else |
| 1533 | if (StoreJ && LoadI && alias(J, I)) { |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1534 | FoundSequentialDependence = true; |
| 1535 | break; |
| 1536 | } |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1537 | |
| 1538 | if (!StoreJ) |
| 1539 | if (!LoadJ || (!LoadI && !StoreI)) { |
| 1540 | // If J is neither load nor store, assume a dependency. |
| 1541 | // If J is a load, but I is neither, also assume a dependency. |
| 1542 | FoundSequentialDependence = true; |
| 1543 | break; |
| 1544 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1545 | // Store followed by store: not OK on V2. |
| 1546 | // Store followed by load: not OK on all. |
| 1547 | // Load followed by store: OK on all. |
| 1548 | // Load followed by load: OK on all. |
| 1549 | continue; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1550 | } |
| 1551 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1552 | // For V4, special case ALLOCFRAME. Even though there is dependency |
| 1553 | // between ALLOCFRAME and subsequent store, allow it to be packetized |
| 1554 | // in a same packet. This implies that the store is using the caller's |
| 1555 | // SP. Hence, offset needs to be updated accordingly. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1556 | if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) { |
| 1557 | unsigned Opc = I.getOpcode(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1558 | switch (Opc) { |
| 1559 | case Hexagon::S2_storerd_io: |
| 1560 | case Hexagon::S2_storeri_io: |
| 1561 | case Hexagon::S2_storerh_io: |
| 1562 | case Hexagon::S2_storerb_io: |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1563 | if (I.getOperand(0).getReg() == HRI->getStackRegister()) { |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 1564 | // Since this store is to be glued with allocframe in the same |
| 1565 | // packet, it will use SP of the previous stack frame, i.e. |
| 1566 | // caller's SP. Therefore, we need to recalculate offset |
| 1567 | // according to this change. |
| 1568 | GlueAllocframeStore = useCallersSP(I); |
| 1569 | if (GlueAllocframeStore) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1570 | continue; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1571 | } |
| 1572 | default: |
| 1573 | break; |
| 1574 | } |
| 1575 | } |
| 1576 | |
Krzysztof Parzyszek | adb7ff0 | 2016-05-06 19:13:38 +0000 | [diff] [blame] | 1577 | // There are certain anti-dependencies that cannot be ignored. |
| 1578 | // Specifically: |
Francis Visoiu Mistrih | a8a83d1 | 2017-12-07 10:40:31 +0000 | [diff] [blame] | 1579 | // J2_call ... implicit-def %r0 ; SUJ |
Krzysztof Parzyszek | adb7ff0 | 2016-05-06 19:13:38 +0000 | [diff] [blame] | 1580 | // R0 = ... ; SUI |
| 1581 | // Those cannot be packetized together, since the call will observe |
| 1582 | // the effect of the assignment to R0. |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 1583 | if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) { |
Krzysztof Parzyszek | adb7ff0 | 2016-05-06 19:13:38 +0000 | [diff] [blame] | 1584 | // Check if I defines any volatile register. We should also check |
| 1585 | // registers that the call may read, but these happen to be a |
| 1586 | // subset of the volatile register set. |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 1587 | for (const MachineOperand &Op : I.operands()) { |
| 1588 | if (Op.isReg() && Op.isDef()) { |
| 1589 | unsigned R = Op.getReg(); |
| 1590 | if (!J.readsRegister(R, HRI) && !J.modifiesRegister(R, HRI)) |
| 1591 | continue; |
| 1592 | } else if (!Op.isRegMask()) { |
| 1593 | // If I has a regmask assume dependency. |
Krzysztof Parzyszek | adb7ff0 | 2016-05-06 19:13:38 +0000 | [diff] [blame] | 1594 | continue; |
Krzysztof Parzyszek | 1aaf41a | 2017-02-17 22:14:51 +0000 | [diff] [blame] | 1595 | } |
Krzysztof Parzyszek | adb7ff0 | 2016-05-06 19:13:38 +0000 | [diff] [blame] | 1596 | FoundSequentialDependence = true; |
| 1597 | break; |
| 1598 | } |
| 1599 | } |
| 1600 | |
| 1601 | // Skip over remaining anti-dependences. Two instructions that are |
| 1602 | // anti-dependent can share a packet, since in most such cases all |
| 1603 | // operands are read before any modifications take place. |
| 1604 | // The exceptions are branch and call instructions, since they are |
| 1605 | // executed after all other instructions have completed (at least |
| 1606 | // conceptually). |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1607 | if (DepType != SDep::Anti) { |
| 1608 | FoundSequentialDependence = true; |
| 1609 | break; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1610 | } |
| 1611 | } |
| 1612 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1613 | if (FoundSequentialDependence) { |
| 1614 | Dependence = true; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1615 | return false; |
| 1616 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1617 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1618 | return true; |
| 1619 | } |
| 1620 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1621 | bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1622 | assert(SUI->getInstr() && SUJ->getInstr()); |
| 1623 | MachineInstr &I = *SUI->getInstr(); |
| 1624 | MachineInstr &J = *SUJ->getInstr(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1625 | |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1626 | bool Coexist = !cannotCoexist(I, J); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1627 | |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1628 | if (Coexist && !Dependence) |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1629 | return true; |
| 1630 | |
| 1631 | // Check if the instruction was promoted to a dot-new. If so, demote it |
| 1632 | // back into a dot-old. |
| 1633 | if (PromotedToDotNew) |
| 1634 | demoteToDotOld(I); |
| 1635 | |
| 1636 | cleanUpDotCur(); |
| 1637 | // Check if the instruction (must be a store) was glued with an allocframe |
| 1638 | // instruction. If so, restore its offset to its original value, i.e. use |
| 1639 | // current SP instead of caller's SP. |
| 1640 | if (GlueAllocframeStore) { |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 1641 | useCalleesSP(I); |
| 1642 | GlueAllocframeStore = false; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1643 | } |
Krzysztof Parzyszek | 8f174dd | 2017-10-11 15:51:44 +0000 | [diff] [blame] | 1644 | |
| 1645 | if (ChangedOffset != INT64_MAX) |
| 1646 | undoChangedOffset(I); |
Krzysztof Parzyszek | c4a9a8d | 2017-10-11 21:20:43 +0000 | [diff] [blame] | 1647 | |
| 1648 | if (GlueToNewValueJump) { |
| 1649 | // Putting I and J together would prevent the new-value jump from being |
| 1650 | // packetized with the producer. In that case I and J must be separated. |
| 1651 | GlueToNewValueJump = false; |
| 1652 | return false; |
| 1653 | } |
| 1654 | |
| 1655 | if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) { |
Krzysztof Parzyszek | 8f174dd | 2017-10-11 15:51:44 +0000 | [diff] [blame] | 1656 | FoundSequentialDependence = false; |
| 1657 | Dependence = false; |
| 1658 | return true; |
| 1659 | } |
| 1660 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1661 | return false; |
| 1662 | } |
| 1663 | |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1664 | |
| 1665 | bool HexagonPacketizerList::foundLSInPacket() { |
| 1666 | bool FoundLoad = false; |
| 1667 | bool FoundStore = false; |
| 1668 | |
| 1669 | for (auto MJ : CurrentPacketMIs) { |
| 1670 | unsigned Opc = MJ->getOpcode(); |
| 1671 | if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe) |
| 1672 | continue; |
| 1673 | if (HII->isMemOp(*MJ)) |
| 1674 | continue; |
| 1675 | if (MJ->mayLoad()) |
| 1676 | FoundLoad = true; |
| 1677 | if (MJ->mayStore() && !HII->isNewValueStore(*MJ)) |
| 1678 | FoundStore = true; |
| 1679 | } |
| 1680 | return FoundLoad && FoundStore; |
| 1681 | } |
| 1682 | |
| 1683 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1684 | MachineBasicBlock::iterator |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1685 | HexagonPacketizerList::addToPacket(MachineInstr &MI) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1686 | MachineBasicBlock::iterator MII = MI.getIterator(); |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1687 | MachineBasicBlock *MBB = MI.getParent(); |
Krzysztof Parzyszek | 9aaf923 | 2017-05-02 18:12:19 +0000 | [diff] [blame] | 1688 | |
Eugene Zelenko | 3b87336 | 2017-09-28 22:27:31 +0000 | [diff] [blame] | 1689 | if (CurrentPacketMIs.empty()) |
Krzysztof Parzyszek | 9aaf923 | 2017-05-02 18:12:19 +0000 | [diff] [blame] | 1690 | PacketStalls = false; |
| 1691 | PacketStalls |= producesStall(MI); |
| 1692 | |
Krzysztof Parzyszek | b7e54e8 | 2018-04-06 18:19:22 +0000 | [diff] [blame] | 1693 | if (MI.isImplicitDef()) { |
| 1694 | // Add to the packet to allow subsequent instructions to be checked |
| 1695 | // properly. |
| 1696 | CurrentPacketMIs.push_back(&MI); |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1697 | return MII; |
Krzysztof Parzyszek | b7e54e8 | 2018-04-06 18:19:22 +0000 | [diff] [blame] | 1698 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1699 | assert(ResourceTracker->canReserveResources(MI)); |
| 1700 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1701 | bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1702 | bool Good = true; |
| 1703 | |
| 1704 | if (GlueToNewValueJump) { |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1705 | MachineInstr &NvjMI = *++MII; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1706 | // We need to put both instructions in the same packet: MI and NvjMI. |
| 1707 | // Either of them can require a constant extender. Try to add both to |
| 1708 | // the current packet, and if that fails, end the packet and start a |
| 1709 | // new one. |
| 1710 | ResourceTracker->reserveResources(MI); |
| 1711 | if (ExtMI) |
| 1712 | Good = tryAllocateResourcesForConstExt(true); |
| 1713 | |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1714 | bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1715 | if (Good) { |
| 1716 | if (ResourceTracker->canReserveResources(NvjMI)) |
| 1717 | ResourceTracker->reserveResources(NvjMI); |
| 1718 | else |
| 1719 | Good = false; |
| 1720 | } |
| 1721 | if (Good && ExtNvjMI) |
| 1722 | Good = tryAllocateResourcesForConstExt(true); |
| 1723 | |
| 1724 | if (!Good) { |
| 1725 | endPacket(MBB, MI); |
| 1726 | assert(ResourceTracker->canReserveResources(MI)); |
| 1727 | ResourceTracker->reserveResources(MI); |
| 1728 | if (ExtMI) { |
| 1729 | assert(canReserveResourcesForConstExt()); |
| 1730 | tryAllocateResourcesForConstExt(true); |
| 1731 | } |
| 1732 | assert(ResourceTracker->canReserveResources(NvjMI)); |
| 1733 | ResourceTracker->reserveResources(NvjMI); |
| 1734 | if (ExtNvjMI) { |
| 1735 | assert(canReserveResourcesForConstExt()); |
| 1736 | reserveResourcesForConstExt(); |
| 1737 | } |
| 1738 | } |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1739 | CurrentPacketMIs.push_back(&MI); |
| 1740 | CurrentPacketMIs.push_back(&NvjMI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1741 | return MII; |
| 1742 | } |
| 1743 | |
| 1744 | ResourceTracker->reserveResources(MI); |
| 1745 | if (ExtMI && !tryAllocateResourcesForConstExt(true)) { |
| 1746 | endPacket(MBB, MI); |
| 1747 | if (PromotedToDotNew) |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1748 | demoteToDotOld(MI); |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 1749 | if (GlueAllocframeStore) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1750 | useCalleesSP(MI); |
Krzysztof Parzyszek | 3b4682f | 2016-07-26 14:24:46 +0000 | [diff] [blame] | 1751 | GlueAllocframeStore = false; |
| 1752 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1753 | ResourceTracker->reserveResources(MI); |
| 1754 | reserveResourcesForConstExt(); |
| 1755 | } |
| 1756 | |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1757 | CurrentPacketMIs.push_back(&MI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1758 | return MII; |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1759 | } |
| 1760 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1761 | void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB, |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1762 | MachineBasicBlock::iterator MI) { |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1763 | // Replace VLIWPacketizerList::endPacket(MBB, MI). |
| 1764 | |
| 1765 | bool memShufDisabled = getmemShufDisabled(); |
| 1766 | if (memShufDisabled && !foundLSInPacket()) { |
| 1767 | setmemShufDisabled(false); |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 1768 | LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n"); |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1769 | } |
| 1770 | memShufDisabled = getmemShufDisabled(); |
| 1771 | |
| 1772 | if (CurrentPacketMIs.size() > 1) { |
| 1773 | MachineBasicBlock::instr_iterator FirstMI(CurrentPacketMIs.front()); |
| 1774 | MachineBasicBlock::instr_iterator LastMI(MI.getInstrIterator()); |
| 1775 | finalizeBundle(*MBB, FirstMI, LastMI); |
| 1776 | |
| 1777 | auto BundleMII = std::prev(FirstMI); |
| 1778 | if (memShufDisabled) |
| 1779 | HII->setBundleNoShuf(BundleMII); |
| 1780 | |
| 1781 | setmemShufDisabled(false); |
| 1782 | } |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1783 | OldPacketMIs = CurrentPacketMIs; |
Krzysztof Parzyszek | a8ab1b7 | 2017-12-11 18:57:54 +0000 | [diff] [blame] | 1784 | CurrentPacketMIs.clear(); |
| 1785 | |
| 1786 | ResourceTracker->clearResources(); |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 1787 | LLVM_DEBUG(dbgs() << "End packet\n"); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1788 | } |
| 1789 | |
Duncan P. N. Exon Smith | 5702287 | 2016-02-27 19:09:00 +0000 | [diff] [blame] | 1790 | bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) { |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1791 | return !producesStall(MI); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1792 | } |
| 1793 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1794 | // V60 forward scheduling. |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1795 | bool HexagonPacketizerList::producesStall(const MachineInstr &I) { |
Krzysztof Parzyszek | 9aaf923 | 2017-05-02 18:12:19 +0000 | [diff] [blame] | 1796 | // If the packet already stalls, then ignore the stall from a subsequent |
| 1797 | // instruction in the same packet. |
| 1798 | if (PacketStalls) |
| 1799 | return false; |
| 1800 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1801 | // Check whether the previous packet is in a different loop. If this is the |
| 1802 | // case, there is little point in trying to avoid a stall because that would |
| 1803 | // favor the rare case (loop entry) over the common case (loop iteration). |
| 1804 | // |
| 1805 | // TODO: We should really be able to check all the incoming edges if this is |
| 1806 | // the first packet in a basic block, so we can avoid stalls from the loop |
| 1807 | // backedge. |
| 1808 | if (!OldPacketMIs.empty()) { |
| 1809 | auto *OldBB = OldPacketMIs.front()->getParent(); |
Krzysztof Parzyszek | f0b34a5 | 2016-07-29 21:49:42 +0000 | [diff] [blame] | 1810 | auto *ThisBB = I.getParent(); |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1811 | if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB)) |
| 1812 | return false; |
| 1813 | } |
| 1814 | |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1815 | SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)]; |
Krzysztof Parzyszek | 9aaf923 | 2017-05-02 18:12:19 +0000 | [diff] [blame] | 1816 | |
Krzysztof Parzyszek | aca8f32 | 2018-04-06 18:13:11 +0000 | [diff] [blame] | 1817 | // If the latency is 0 and there is a data dependence between this |
| 1818 | // instruction and any instruction in the current packet, we disregard any |
| 1819 | // potential stalls due to the instructions in the previous packet. Most of |
| 1820 | // the instruction pairs that can go together in the same packet have 0 |
| 1821 | // latency between them. The exceptions are |
| 1822 | // 1. NewValueJumps as they're generated much later and the latencies can't |
| 1823 | // be changed at that point. |
| 1824 | // 2. .cur instructions, if its consumer has a 0 latency successor (such as |
| 1825 | // .new). In this case, the latency between .cur and the consumer stays |
| 1826 | // non-zero even though we can have both .cur and .new in the same packet. |
| 1827 | // Changing the latency to 0 is not an option as it causes software pipeliner |
| 1828 | // to not pipeline in some cases. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1829 | |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1830 | // For Example: |
| 1831 | // { |
| 1832 | // I1: v6.cur = vmem(r0++#1) |
| 1833 | // I2: v7 = valign(v6,v4,r2) |
| 1834 | // I3: vmem(r5++#1) = v7.new |
| 1835 | // } |
| 1836 | // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2. |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1837 | |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1838 | for (auto J : CurrentPacketMIs) { |
| 1839 | SUnit *SUJ = MIToSUnit[J]; |
| 1840 | for (auto &Pred : SUI->Preds) |
Krzysztof Parzyszek | aca8f32 | 2018-04-06 18:13:11 +0000 | [diff] [blame] | 1841 | if (Pred.getSUnit() == SUJ) |
| 1842 | if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) || |
| 1843 | HII->isNewValueJump(I) || HII->isToBeScheduledASAP(*J, I)) |
| 1844 | return false; |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1845 | } |
| 1846 | |
Krzysztof Parzyszek | 9aaf923 | 2017-05-02 18:12:19 +0000 | [diff] [blame] | 1847 | // Check if the latency is greater than one between this instruction and any |
| 1848 | // instruction in the previous packet. |
Krzysztof Parzyszek | 2af5037 | 2017-05-03 20:10:36 +0000 | [diff] [blame] | 1849 | for (auto J : OldPacketMIs) { |
| 1850 | SUnit *SUJ = MIToSUnit[J]; |
| 1851 | for (auto &Pred : SUI->Preds) |
| 1852 | if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1) |
| 1853 | return true; |
| 1854 | } |
| 1855 | |
Krzysztof Parzyszek | 56bbf54 | 2015-12-16 19:36:12 +0000 | [diff] [blame] | 1856 | return false; |
| 1857 | } |
| 1858 | |
Sirish Pande | f8e5e3c | 2012-05-03 21:52:53 +0000 | [diff] [blame] | 1859 | //===----------------------------------------------------------------------===// |
| 1860 | // Public Constructor Functions |
| 1861 | //===----------------------------------------------------------------------===// |
| 1862 | |
| 1863 | FunctionPass *llvm::createHexagonPacketizer() { |
| 1864 | return new HexagonPacketizer(); |
| 1865 | } |