blob: 56ab69db9bd1a3aa2df4bf9549769a02ddad626f [file] [log] [blame]
Eugene Zelenko3b873362017-09-28 22:27:31 +00001//===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===//
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00002//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements a simple VLIW packetizer using DFA. The packetizer works on
11// machine basic blocks. For each instruction I in BB, the packetizer consults
12// the DFA to see if machine resources are available to execute I. If so, the
13// packetizer checks if I depends on any instruction J in the current packet.
14// If no dependency is found, I is added to current packet and machine resource
15// is marked as taken. If any dependency is found, a target API call is made to
16// prune the dependence.
17//
18//===----------------------------------------------------------------------===//
Eugene Zelenko3b873362017-09-28 22:27:31 +000019
Chandler Carruth6bda14b2017-06-06 11:49:48 +000020#include "HexagonVLIWPacketizer.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000021#include "Hexagon.h"
22#include "HexagonInstrInfo.h"
Jyotsna Verma84256432013-03-01 17:37:13 +000023#include "HexagonRegisterInfo.h"
24#include "HexagonSubtarget.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000025#include "llvm/ADT/BitVector.h"
26#include "llvm/ADT/DenseSet.h"
27#include "llvm/ADT/STLExtras.h"
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +000028#include "llvm/Analysis/AliasAnalysis.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000029#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000031#include "llvm/CodeGen/MachineDominators.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000032#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000034#include "llvm/CodeGen/MachineFunctionPass.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000035#include "llvm/CodeGen/MachineInstr.h"
Matthias Braunf8422972017-12-13 02:51:04 +000036#include "llvm/CodeGen/MachineInstrBundle.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000037#include "llvm/CodeGen/MachineLoopInfo.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000038#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/ScheduleDAG.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000040#include "llvm/CodeGen/TargetRegisterInfo.h"
41#include "llvm/CodeGen/TargetSubtargetInfo.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000042#include "llvm/IR/DebugLoc.h"
43#include "llvm/MC/MCInstrDesc.h"
44#include "llvm/Pass.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000045#include "llvm/Support/CommandLine.h"
Chandler Carruth8a8cd2b2014-01-07 11:48:04 +000046#include "llvm/Support/Debug.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000047#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/raw_ostream.h"
Eugene Zelenko3b873362017-09-28 22:27:31 +000049#include <cassert>
50#include <cstdint>
51#include <iterator>
Sirish Pandef8e5e3c2012-05-03 21:52:53 +000052
53using namespace llvm;
54
Chandler Carruth84e68b22014-04-22 02:41:26 +000055#define DEBUG_TYPE "packets"
56
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +000057static cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden,
58 cl::ZeroOrMore, cl::init(false),
59 cl::desc("Disable Hexagon packetizer pass"));
60
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +000061cl::opt<bool> Slot1Store("slot1-store-slot0-load", cl::Hidden,
62 cl::ZeroOrMore, cl::init(true),
63 cl::desc("Allow slot1 store and slot0 load"));
64
Jyotsna Verma1d297502013-05-02 15:39:30 +000065static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles",
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +000066 cl::ZeroOrMore, cl::Hidden, cl::init(true),
67 cl::desc("Allow non-solo packetization of volatile memory references"));
68
69static cl::opt<bool> EnableGenAllInsnClass("enable-gen-insn", cl::init(false),
70 cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC"));
71
72static cl::opt<bool> DisableVecDblNVStores("disable-vecdbl-nv-stores",
73 cl::init(false), cl::Hidden, cl::ZeroOrMore,
74 cl::desc("Disable vector double new-value-stores"));
75
76extern cl::opt<bool> ScheduleInlineAsm;
Jyotsna Verma1d297502013-05-02 15:39:30 +000077
Jyotsna Verma1d297502013-05-02 15:39:30 +000078namespace llvm {
Jyotsna Verma1d297502013-05-02 15:39:30 +000079
Eugene Zelenko3b873362017-09-28 22:27:31 +000080FunctionPass *createHexagonPacketizer();
81void initializeHexagonPacketizerPass(PassRegistry&);
82
83} // end namespace llvm
Jyotsna Verma1d297502013-05-02 15:39:30 +000084
Sirish Pandef8e5e3c2012-05-03 21:52:53 +000085namespace {
Eugene Zelenko3b873362017-09-28 22:27:31 +000086
Sirish Pandef8e5e3c2012-05-03 21:52:53 +000087 class HexagonPacketizer : public MachineFunctionPass {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +000088 public:
89 static char ID;
Eugene Zelenko3b873362017-09-28 22:27:31 +000090
Krzysztof Parzyszekdf4a05d2017-07-10 18:38:52 +000091 HexagonPacketizer() : MachineFunctionPass(ID) {}
Sirish Pandef8e5e3c2012-05-03 21:52:53 +000092
Craig Topper906c2cd2014-04-29 07:58:16 +000093 void getAnalysisUsage(AnalysisUsage &AU) const override {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +000094 AU.setPreservesCFG();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +000095 AU.addRequired<AAResultsWrapperPass>();
Jyotsna Verma1d297502013-05-02 15:39:30 +000096 AU.addRequired<MachineBranchProbabilityInfo>();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +000097 AU.addRequired<MachineDominatorTree>();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +000098 AU.addRequired<MachineLoopInfo>();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +000099 AU.addPreserved<MachineDominatorTree>();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000100 AU.addPreserved<MachineLoopInfo>();
101 MachineFunctionPass::getAnalysisUsage(AU);
102 }
Eugene Zelenko3b873362017-09-28 22:27:31 +0000103
Mehdi Amini117296c2016-10-01 02:56:57 +0000104 StringRef getPassName() const override { return "Hexagon Packetizer"; }
Craig Topper906c2cd2014-04-29 07:58:16 +0000105 bool runOnMachineFunction(MachineFunction &Fn) override;
Eugene Zelenko3b873362017-09-28 22:27:31 +0000106
Derek Schuff1dbf7a52016-04-04 17:09:25 +0000107 MachineFunctionProperties getRequiredProperties() const override {
108 return MachineFunctionProperties().set(
Matthias Braun1eb47362016-08-25 01:27:13 +0000109 MachineFunctionProperties::Property::NoVRegs);
Derek Schuff1dbf7a52016-04-04 17:09:25 +0000110 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000111
112 private:
113 const HexagonInstrInfo *HII;
114 const HexagonRegisterInfo *HRI;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000115 };
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000116
Eugene Zelenko3b873362017-09-28 22:27:31 +0000117} // end anonymous namespace
118
119char HexagonPacketizer::ID = 0;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000120
Krzysztof Parzyszekdf4a05d2017-07-10 18:38:52 +0000121INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer",
122 "Hexagon Packetizer", false, false)
Jyotsna Verma1d297502013-05-02 15:39:30 +0000123INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
124INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
125INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
Chandler Carruth7b560d42015-09-09 17:55:00 +0000126INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
Krzysztof Parzyszekdf4a05d2017-07-10 18:38:52 +0000127INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer",
128 "Hexagon Packetizer", false, false)
Jyotsna Verma1d297502013-05-02 15:39:30 +0000129
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000130HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF,
131 MachineLoopInfo &MLI, AliasAnalysis *AA,
132 const MachineBranchProbabilityInfo *MBPI)
133 : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI) {
134 HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
135 HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
Krzysztof Parzyszek9be66732016-07-15 17:48:09 +0000136
Eugene Zelenko3b873362017-09-28 22:27:31 +0000137 addMutation(llvm::make_unique<HexagonSubtarget::UsrOverflowMutation>());
138 addMutation(llvm::make_unique<HexagonSubtarget::HVXMemLatencyMutation>());
139 addMutation(llvm::make_unique<HexagonSubtarget::BankConflictMutation>());
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000140}
141
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000142// Check if FirstI modifies a register that SecondI reads.
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +0000143static bool hasWriteToReadDep(const MachineInstr &FirstI,
144 const MachineInstr &SecondI,
145 const TargetRegisterInfo *TRI) {
146 for (auto &MO : FirstI.operands()) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000147 if (!MO.isReg() || !MO.isDef())
148 continue;
149 unsigned R = MO.getReg();
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +0000150 if (SecondI.readsRegister(R, TRI))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000151 return true;
152 }
153 return false;
154}
155
156
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000157static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000158 MachineBasicBlock::iterator BundleIt, bool Before) {
159 MachineBasicBlock::instr_iterator InsertPt;
160 if (Before)
Duncan P. N. Exon Smithd84f6002016-02-22 21:30:15 +0000161 InsertPt = BundleIt.getInstrIterator();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000162 else
Duncan P. N. Exon Smithd84f6002016-02-22 21:30:15 +0000163 InsertPt = std::next(BundleIt).getInstrIterator();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000164
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000165 MachineBasicBlock &B = *MI.getParent();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000166 // The instruction should at least be bundled with the preceding instruction
167 // (there will always be one, i.e. BUNDLE, if nothing else).
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000168 assert(MI.isBundledWithPred());
169 if (MI.isBundledWithSucc()) {
170 MI.clearFlag(MachineInstr::BundledSucc);
171 MI.clearFlag(MachineInstr::BundledPred);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000172 } else {
173 // If it's not bundled with the successor (i.e. it is the last one
174 // in the bundle), then we can simply unbundle it from the predecessor,
175 // which will take care of updating the predecessor's flag.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000176 MI.unbundleFromPred();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000177 }
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000178 B.splice(InsertPt, &B, MI.getIterator());
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000179
180 // Get the size of the bundle without asserting.
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +0000181 MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000182 MachineBasicBlock::const_instr_iterator E = B.instr_end();
183 unsigned Size = 0;
184 for (++I; I != E && I->isBundledWithPred(); ++I)
185 ++Size;
186
187 // If there are still two or more instructions, then there is nothing
188 // else to be done.
189 if (Size > 1)
190 return BundleIt;
191
192 // Otherwise, extract the single instruction out and delete the bundle.
193 MachineBasicBlock::iterator NextIt = std::next(BundleIt);
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000194 MachineInstr &SingleI = *BundleIt->getNextNode();
195 SingleI.unbundleFromPred();
196 assert(!SingleI.isBundledWithSucc());
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000197 BundleIt->eraseFromParent();
198 return NextIt;
199}
200
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000201bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {
Krzysztof Parzyszek5d41cc12018-03-12 17:47:46 +0000202 auto &HST = MF.getSubtarget<HexagonSubtarget>();
203 if (DisablePacketizer || !HST.usePackets() || skipFunction(MF.getFunction()))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000204 return false;
205
Krzysztof Parzyszek5d41cc12018-03-12 17:47:46 +0000206 HII = HST.getInstrInfo();
207 HRI = HST.getRegisterInfo();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000208 auto &MLI = getAnalysis<MachineLoopInfo>();
209 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
210 auto *MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
211
212 if (EnableGenAllInsnClass)
213 HII->genAllInsnTimingClasses(MF);
214
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000215 // Instantiate the packetizer.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000216 HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000217
218 // DFA state table should not be empty.
219 assert(Packetizer.getResourceTracker() && "Empty DFA table!");
220
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000221 // Loop over all basic blocks and remove KILL pseudo-instructions
222 // These instructions confuse the dependence analysis. Consider:
223 // D0 = ... (Insn 0)
224 // R0 = KILL R0, D0 (Insn 1)
225 // R0 = ... (Insn 2)
226 // Here, Insn 1 will result in the dependence graph not emitting an output
227 // dependence between Insn 0 and Insn 2. This can lead to incorrect
228 // packetization
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000229 for (auto &MB : MF) {
230 auto End = MB.end();
231 auto MI = MB.begin();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000232 while (MI != End) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000233 auto NextI = std::next(MI);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000234 if (MI->isKill()) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000235 MB.erase(MI);
236 End = MB.end();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000237 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000238 MI = NextI;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000239 }
240 }
241
242 // Loop over all of the basic blocks.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000243 for (auto &MB : MF) {
244 auto Begin = MB.begin(), End = MB.end();
245 while (Begin != End) {
Krzysztof Parzyszeke3ec97b2017-05-24 13:43:42 +0000246 // Find the first non-boundary starting from the end of the last
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000247 // scheduling region.
248 MachineBasicBlock::iterator RB = Begin;
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000249 while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000250 ++RB;
Krzysztof Parzyszeke3ec97b2017-05-24 13:43:42 +0000251 // Find the first boundary starting from the beginning of the new
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000252 // region.
253 MachineBasicBlock::iterator RE = RB;
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000254 while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000255 ++RE;
256 // Add the scheduling boundary if it's not block end.
257 if (RE != End)
258 ++RE;
259 // If RB == End, then RE == End.
260 if (RB != End)
261 Packetizer.PacketizeMIs(&MB, RB, RE);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000262
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000263 Begin = RE;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000264 }
265 }
266
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000267 Packetizer.unpacketizeSoloInstrs(MF);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000268 return true;
269}
270
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000271// Reserve resources for a constant extender. Trigger an assertion if the
272// reservation fails.
273void HexagonPacketizerList::reserveResourcesForConstExt() {
274 if (!tryAllocateResourcesForConstExt(true))
275 llvm_unreachable("Resources not available");
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000276}
277
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000278bool HexagonPacketizerList::canReserveResourcesForConstExt() {
279 return tryAllocateResourcesForConstExt(false);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000280}
281
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000282// Allocate resources (i.e. 4 bytes) for constant extender. If succeeded,
283// return true, otherwise, return false.
284bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) {
285 auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc());
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +0000286 bool Avail = ResourceTracker->canReserveResources(*ExtMI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000287 if (Reserve && Avail)
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +0000288 ResourceTracker->reserveResources(*ExtMI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000289 MF.DeleteMachineInstr(ExtMI);
290 return Avail;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000291}
292
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000293bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000294 SDep::Kind DepType, unsigned DepReg) {
295 // Check for LR dependence.
296 if (DepReg == HRI->getRARegister())
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000297 return true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000298
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000299 if (HII->isDeallocRet(MI))
300 if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000301 return true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000302
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000303 // Call-like instructions can be packetized with preceding instructions
304 // that define registers implicitly used or modified by the call. Explicit
305 // uses are still prohibited, as in the case of indirect calls:
306 // r0 = ...
307 // J2_jumpr r0
308 if (DepType == SDep::Data) {
309 for (const MachineOperand MO : MI.operands())
310 if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit())
311 return true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000312 }
313
314 return false;
315}
316
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000317static bool isRegDependence(const SDep::Kind DepType) {
318 return DepType == SDep::Data || DepType == SDep::Anti ||
319 DepType == SDep::Output;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000320}
321
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000322static bool isDirectJump(const MachineInstr &MI) {
323 return MI.getOpcode() == Hexagon::J2_jump;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000324}
325
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000326static bool isSchedBarrier(const MachineInstr &MI) {
327 switch (MI.getOpcode()) {
Colin LeMahieub882f2b2015-02-05 18:56:28 +0000328 case Hexagon::Y2_barrier:
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000329 return true;
330 }
331 return false;
332}
333
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000334static bool isControlFlow(const MachineInstr &MI) {
335 return MI.getDesc().isTerminator() || MI.getDesc().isCall();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000336}
337
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000338/// Returns true if the instruction modifies a callee-saved register.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000339static bool doesModifyCalleeSavedReg(const MachineInstr &MI,
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000340 const TargetRegisterInfo *TRI) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000341 const MachineFunction &MF = *MI.getParent()->getParent();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000342 for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000343 if (MI.modifiesRegister(*CSR, TRI))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000344 return true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000345 return false;
346}
347
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000348// Returns true if an instruction can be promoted to .new predicate or
349// new-value store.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000350bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI,
Krzysztof Parzyszek2a480592016-07-26 20:30:30 +0000351 const TargetRegisterClass *NewRC) {
352 // Vector stores can be predicated, and can be new-value stores, but
353 // they cannot be predicated on a .new predicate value.
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000354 if (NewRC == &Hexagon::PredRegsRegClass) {
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +0000355 if (HII->isHVXVec(MI) && MI.mayStore())
Krzysztof Parzyszek2a480592016-07-26 20:30:30 +0000356 return false;
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000357 return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, nullptr) > 0;
358 }
359 // If the class is not PredRegs, it could only apply to new-value stores.
360 return HII->mayBeNewStore(MI);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000361}
362
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000363// Promote an instructiont to its .cur form.
364// At this time, we have already made a call to canPromoteToDotCur and made
365// sure that it can *indeed* be promoted.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000366bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000367 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
368 const TargetRegisterClass* RC) {
369 assert(DepType == SDep::Data);
370 int CurOpcode = HII->getDotCurOp(MI);
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000371 MI.setDesc(HII->get(CurOpcode));
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000372 return true;
373}
374
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000375void HexagonPacketizerList::cleanUpDotCur() {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000376 MachineInstr *MI = nullptr;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000377 for (auto BI : CurrentPacketMIs) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000378 LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););
Krzysztof Parzyszek0a8043e2017-05-03 15:28:56 +0000379 if (HII->isDotCurInst(*BI)) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000380 MI = BI;
381 continue;
382 }
383 if (MI) {
384 for (auto &MO : BI->operands())
385 if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg())
386 return;
387 }
388 }
389 if (!MI)
390 return;
391 // We did not find a use of the CUR, so de-cur it.
Krzysztof Parzyszek0a8043e2017-05-03 15:28:56 +0000392 MI->setDesc(HII->get(HII->getNonDotCurOp(*MI)));
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000393 LLVM_DEBUG(dbgs() << "Demoted CUR "; MI->dump(););
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000394}
395
396// Check to see if an instruction can be dot cur.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000397bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000398 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
399 const TargetRegisterClass *RC) {
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +0000400 if (!HII->isHVXVec(MI))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000401 return false;
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +0000402 if (!HII->isHVXVec(*MII))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000403 return false;
404
405 // Already a dot new instruction.
406 if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))
407 return false;
408
409 if (!HII->mayBeCurLoad(MI))
410 return false;
411
412 // The "cur value" cannot come from inline asm.
413 if (PacketSU->getInstr()->isInlineAsm())
414 return false;
415
416 // Make sure candidate instruction uses cur.
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000417 LLVM_DEBUG(dbgs() << "Can we DOT Cur Vector MI\n"; MI.dump();
418 dbgs() << "in packet\n";);
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +0000419 MachineInstr &MJ = *MII;
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000420 LLVM_DEBUG({
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +0000421 dbgs() << "Checking CUR against ";
422 MJ.dump();
423 });
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000424 unsigned DestReg = MI.getOperand(0).getReg();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000425 bool FoundMatch = false;
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +0000426 for (auto &MO : MJ.operands())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000427 if (MO.isReg() && MO.getReg() == DestReg)
428 FoundMatch = true;
429 if (!FoundMatch)
430 return false;
431
432 // Check for existing uses of a vector register within the packet which
433 // would be affected by converting a vector load into .cur formt.
434 for (auto BI : CurrentPacketMIs) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000435 LLVM_DEBUG(dbgs() << "packet has "; BI->dump(););
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000436 if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo()))
437 return false;
438 }
439
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000440 LLVM_DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump(););
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000441 // We can convert the opcode into a .cur.
442 return true;
443}
444
445// Promote an instruction to its .new form. At this time, we have already
446// made a call to canPromoteToDotNew and made sure that it can *indeed* be
447// promoted.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000448bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000449 SDep::Kind DepType, MachineBasicBlock::iterator &MII,
450 const TargetRegisterClass* RC) {
Eugene Zelenko3b873362017-09-28 22:27:31 +0000451 assert(DepType == SDep::Data);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000452 int NewOpcode;
453 if (RC == &Hexagon::PredRegsRegClass)
454 NewOpcode = HII->getDotNewPredOp(MI, MBPI);
455 else
456 NewOpcode = HII->getDotNewOp(MI);
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000457 MI.setDesc(HII->get(NewOpcode));
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000458 return true;
459}
460
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000461bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) {
Krzysztof Parzyszek143158b2017-03-06 17:03:16 +0000462 int NewOpcode = HII->getDotOldOp(MI);
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000463 MI.setDesc(HII->get(NewOpcode));
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000464 return true;
465}
466
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000467bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) {
468 unsigned Opc = MI.getOpcode();
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +0000469 switch (Opc) {
470 case Hexagon::S2_storerd_io:
471 case Hexagon::S2_storeri_io:
472 case Hexagon::S2_storerh_io:
473 case Hexagon::S2_storerb_io:
474 break;
475 default:
476 llvm_unreachable("Unexpected instruction");
477 }
Matthias Braun941a7052016-07-28 18:40:00 +0000478 unsigned FrameSize = MF.getFrameInfo().getStackSize();
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000479 MachineOperand &Off = MI.getOperand(1);
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +0000480 int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);
Krzysztof Parzyszek55772972017-09-15 15:46:05 +0000481 if (HII->isValidOffset(Opc, NewOff, HRI)) {
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +0000482 Off.setImm(NewOff);
483 return true;
484 }
485 return false;
486}
487
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000488void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) {
489 unsigned Opc = MI.getOpcode();
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +0000490 switch (Opc) {
491 case Hexagon::S2_storerd_io:
492 case Hexagon::S2_storeri_io:
493 case Hexagon::S2_storerh_io:
494 case Hexagon::S2_storerb_io:
495 break;
496 default:
497 llvm_unreachable("Unexpected instruction");
498 }
Matthias Braun941a7052016-07-28 18:40:00 +0000499 unsigned FrameSize = MF.getFrameInfo().getStackSize();
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000500 MachineOperand &Off = MI.getOperand(1);
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +0000501 Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE);
502}
503
Krzysztof Parzyszek8f174dd2017-10-11 15:51:44 +0000504/// Return true if we can update the offset in MI so that MI and MJ
505/// can be packetized together.
506bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) {
507 assert(SUI->getInstr() && SUJ->getInstr());
508 MachineInstr &MI = *SUI->getInstr();
509 MachineInstr &MJ = *SUJ->getInstr();
510
511 unsigned BPI, OPI;
512 if (!HII->getBaseAndOffsetPosition(MI, BPI, OPI))
513 return false;
514 unsigned BPJ, OPJ;
515 if (!HII->getBaseAndOffsetPosition(MJ, BPJ, OPJ))
516 return false;
517 unsigned Reg = MI.getOperand(BPI).getReg();
518 if (Reg != MJ.getOperand(BPJ).getReg())
519 return false;
520 // Make sure that the dependences do not restrict adding MI to the packet.
521 // That is, ignore anti dependences, and make sure the only data dependence
522 // involves the specific register.
523 for (const auto &PI : SUI->Preds)
524 if (PI.getKind() != SDep::Anti &&
525 (PI.getKind() != SDep::Data || PI.getReg() != Reg))
526 return false;
527 int Incr;
528 if (!HII->getIncrementValue(MJ, Incr))
529 return false;
530
531 int64_t Offset = MI.getOperand(OPI).getImm();
Krzysztof Parzyszek0f983d62018-03-30 19:28:37 +0000532 if (!HII->isValidOffset(MI.getOpcode(), Offset+Incr, HRI))
533 return false;
534
Krzysztof Parzyszek8f174dd2017-10-11 15:51:44 +0000535 MI.getOperand(OPI).setImm(Offset + Incr);
536 ChangedOffset = Offset;
537 return true;
538}
539
540/// Undo the changed offset. This is needed if the instruction cannot be
541/// added to the current packet due to a different instruction.
542void HexagonPacketizerList::undoChangedOffset(MachineInstr &MI) {
543 unsigned BP, OP;
544 if (!HII->getBaseAndOffsetPosition(MI, BP, OP))
545 llvm_unreachable("Unable to find base and offset operands.");
546 MI.getOperand(OP).setImm(ChangedOffset);
547}
548
Jyotsna Verma300f0b92013-05-10 20:27:34 +0000549enum PredicateKind {
550 PK_False,
551 PK_True,
552 PK_Unknown
553};
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000554
Jyotsna Verma300f0b92013-05-10 20:27:34 +0000555/// Returns true if an instruction is predicated on p0 and false if it's
556/// predicated on !p0.
Duncan P. N. Exon Smith6307eb52016-02-23 02:46:52 +0000557static PredicateKind getPredicateSense(const MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000558 const HexagonInstrInfo *HII) {
559 if (!HII->isPredicated(MI))
Jyotsna Verma300f0b92013-05-10 20:27:34 +0000560 return PK_Unknown;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000561 if (HII->isPredicatedTrue(MI))
Jyotsna Verma300f0b92013-05-10 20:27:34 +0000562 return PK_True;
Jyotsna Verma300f0b92013-05-10 20:27:34 +0000563 return PK_False;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000564}
565
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000566static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000567 const HexagonInstrInfo *HII) {
Krzysztof Parzyszek8fb181c2016-08-01 17:55:48 +0000568 assert(HII->isPostIncrement(MI) && "Not a post increment operation.");
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000569#ifndef NDEBUG
570 // Post Increment means duplicates. Use dense map to find duplicates in the
571 // list. Caution: Densemap initializes with the minimum of 64 buckets,
572 // whereas there are at most 5 operands in the post increment.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000573 DenseSet<unsigned> DefRegsSet;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000574 for (auto &MO : MI.operands())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000575 if (MO.isReg() && MO.isDef())
576 DefRegsSet.insert(MO.getReg());
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000577
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000578 for (auto &MO : MI.operands())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000579 if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))
580 return MO;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000581#else
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000582 if (MI.mayLoad()) {
583 const MachineOperand &Op1 = MI.getOperand(1);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000584 // The 2nd operand is always the post increment operand in load.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000585 assert(Op1.isReg() && "Post increment operand has be to a register.");
586 return Op1;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000587 }
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000588 if (MI.getDesc().mayStore()) {
589 const MachineOperand &Op0 = MI.getOperand(0);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000590 // The 1st operand is always the post increment operand in store.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000591 assert(Op0.isReg() && "Post increment operand has be to a register.");
592 return Op0;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000593 }
594#endif
595 // we should never come here.
596 llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");
597}
598
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000599// Get the value being stored.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000600static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000601 // value being stored is always the last operand.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000602 return MI.getOperand(MI.getNumOperands()-1);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000603}
604
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000605static bool isLoadAbsSet(const MachineInstr &MI) {
606 unsigned Opc = MI.getOpcode();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000607 switch (Opc) {
608 case Hexagon::L4_loadrd_ap:
609 case Hexagon::L4_loadrb_ap:
610 case Hexagon::L4_loadrh_ap:
611 case Hexagon::L4_loadrub_ap:
612 case Hexagon::L4_loadruh_ap:
613 case Hexagon::L4_loadri_ap:
614 return true;
615 }
616 return false;
617}
618
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000619static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000620 assert(isLoadAbsSet(MI));
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000621 return MI.getOperand(1);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000622}
623
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000624// Can be new value store?
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000625// Following restrictions are to be respected in convert a store into
626// a new value store.
627// 1. If an instruction uses auto-increment, its address register cannot
628// be a new-value register. Arch Spec 5.4.2.1
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000629// 2. If an instruction uses absolute-set addressing mode, its address
630// register cannot be a new-value register. Arch Spec 5.4.2.1.
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000631// 3. If an instruction produces a 64-bit result, its registers cannot be used
632// as new-value registers. Arch Spec 5.4.2.2.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000633// 4. If the instruction that sets the new-value register is conditional, then
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000634// the instruction that uses the new-value register must also be conditional,
635// and both must always have their predicates evaluate identically.
636// Arch Spec 5.4.2.3.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000637// 5. There is an implied restriction that a packet cannot have another store,
638// if there is a new value store in the packet. Corollary: if there is
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000639// already a store in a packet, there can not be a new value store.
640// Arch Spec: 3.4.4.2
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000641bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI,
642 const MachineInstr &PacketMI, unsigned DepReg) {
Jyotsna Verma438cec52013-05-10 20:58:11 +0000643 // Make sure we are looking at the store, that can be promoted.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000644 if (!HII->mayBeNewStore(MI))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000645 return false;
646
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000647 // Make sure there is dependency and can be new value'd.
648 const MachineOperand &Val = getStoreValueOperand(MI);
649 if (Val.isReg() && Val.getReg() != DepReg)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000650 return false;
651
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000652 const MCInstrDesc& MCID = PacketMI.getDesc();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000653
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000654 // First operand is always the result.
655 const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF);
656 // Double regs can not feed into new value store: PRM section: 5.4.2.2.
657 if (PacketRC == &Hexagon::DoubleRegsRegClass)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000658 return false;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000659
660 // New-value stores are of class NV (slot 0), dual stores require class ST
661 // in slot 0 (PRM 5.5).
662 for (auto I : CurrentPacketMIs) {
663 SUnit *PacketSU = MIToSUnit.find(I)->second;
664 if (PacketSU->getInstr()->mayStore())
665 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000666 }
667
668 // Make sure it's NOT the post increment register that we are going to
669 // new value.
Krzysztof Parzyszek8fb181c2016-08-01 17:55:48 +0000670 if (HII->isPostIncrement(MI) &&
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000671 getPostIncrementOperand(MI, HII).getReg() == DepReg) {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000672 return false;
673 }
674
Krzysztof Parzyszek8fb181c2016-08-01 17:55:48 +0000675 if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() &&
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000676 getPostIncrementOperand(PacketMI, HII).getReg() == DepReg) {
677 // If source is post_inc, or absolute-set addressing, it can not feed
678 // into new value store
679 // r3 = memw(r2++#4)
680 // memw(r30 + #-1404) = r2.new -> can not be new value store
681 // arch spec section: 5.4.2.1.
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000682 return false;
683 }
684
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000685 if (isLoadAbsSet(PacketMI) && getAbsSetOperand(PacketMI).getReg() == DepReg)
686 return false;
687
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000688 // If the source that feeds the store is predicated, new value store must
Jyotsna Verma438cec52013-05-10 20:58:11 +0000689 // also be predicated.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000690 if (HII->isPredicated(PacketMI)) {
691 if (!HII->isPredicated(MI))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000692 return false;
693
694 // Check to make sure that they both will have their predicates
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000695 // evaluate identically.
Sirish Pande95d01172012-05-11 20:00:34 +0000696 unsigned predRegNumSrc = 0;
697 unsigned predRegNumDst = 0;
Craig Topper062a2ba2014-04-25 05:30:21 +0000698 const TargetRegisterClass* predRegClass = nullptr;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000699
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000700 // Get predicate register used in the source instruction.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000701 for (auto &MO : PacketMI.operands()) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000702 if (!MO.isReg())
703 continue;
704 predRegNumSrc = MO.getReg();
705 predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc);
706 if (predRegClass == &Hexagon::PredRegsRegClass)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000707 break;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000708 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000709 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
710 "predicate register not found in a predicated PacketMI instruction");
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000711
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000712 // Get predicate register used in new-value store instruction.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000713 for (auto &MO : MI.operands()) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000714 if (!MO.isReg())
715 continue;
716 predRegNumDst = MO.getReg();
717 predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst);
718 if (predRegClass == &Hexagon::PredRegsRegClass)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000719 break;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000720 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000721 assert((predRegClass == &Hexagon::PredRegsRegClass) &&
722 "predicate register not found in a predicated MI instruction");
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000723
724 // New-value register producer and user (store) need to satisfy these
725 // constraints:
726 // 1) Both instructions should be predicated on the same register.
727 // 2) If producer of the new-value register is .new predicated then store
728 // should also be .new predicated and if producer is not .new predicated
729 // then store should not be .new predicated.
730 // 3) Both new-value register producer and user should have same predicate
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000731 // sense, i.e, either both should be negated or both should be non-negated.
732 if (predRegNumDst != predRegNumSrc ||
Duncan P. N. Exon Smith6307eb52016-02-23 02:46:52 +0000733 HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) ||
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000734 getPredicateSense(MI, HII) != getPredicateSense(PacketMI, HII))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000735 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000736 }
737
738 // Make sure that other than the new-value register no other store instruction
739 // register has been modified in the same packet. Predicate registers can be
740 // modified by they should not be modified between the producer and the store
741 // instruction as it will make them both conditional on different values.
742 // We already know this to be true for all the instructions before and
743 // including PacketMI. Howerver, we need to perform the check for the
744 // remaining instructions in the packet.
745
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000746 unsigned StartCheck = 0;
747
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000748 for (auto I : CurrentPacketMIs) {
749 SUnit *TempSU = MIToSUnit.find(I)->second;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000750 MachineInstr &TempMI = *TempSU->getInstr();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000751
752 // Following condition is true for all the instructions until PacketMI is
753 // reached (StartCheck is set to 0 before the for loop).
754 // StartCheck flag is 1 for all the instructions after PacketMI.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000755 if (&TempMI != &PacketMI && !StartCheck) // Start processing only after
756 continue; // encountering PacketMI.
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000757
758 StartCheck = 1;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000759 if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence.
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000760 continue;
761
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000762 for (auto &MO : MI.operands())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000763 if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000764 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000765 }
766
Alp Tokerf907b892013-12-05 05:44:44 +0000767 // Make sure that for non-POST_INC stores:
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000768 // 1. The only use of reg is DepReg and no other registers.
769 // This handles V4 base+index registers.
770 // The following store can not be dot new.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000771 // Eg. r0 = add(r0, #3)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000772 // memw(r1+r0<<#2) = r0
Krzysztof Parzyszek8fb181c2016-08-01 17:55:48 +0000773 if (!HII->isPostIncrement(MI)) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000774 for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {
775 const MachineOperand &MO = MI.getOperand(opNum);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000776 if (MO.isReg() && MO.getReg() == DepReg)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000777 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000778 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000779 }
780
781 // If data definition is because of implicit definition of the register,
782 // do not newify the store. Eg.
Francis Visoiu Mistriha8a83d12017-12-07 10:40:31 +0000783 // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12
784 // S2_storerh_io %r8, 2, killed %r12; mem:ST2[%scevgep343]
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000785 for (auto &MO : PacketMI.operands()) {
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +0000786 if (MO.isRegMask() && MO.clobbersPhysReg(DepReg))
787 return false;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000788 if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())
789 continue;
790 unsigned R = MO.getReg();
791 if (R == DepReg || HRI->isSuperRegister(DepReg, R))
792 return false;
793 }
794
795 // Handle imp-use of super reg case. There is a target independent side
796 // change that should prevent this situation but I am handling it for
797 // just-in-case. For example, we cannot newify R2 in the following case:
Francis Visoiu Mistriha8a83d12017-12-07 10:40:31 +0000798 // %r3 = A2_tfrsi 0;
799 // S2_storeri_io killed %r0, 0, killed %r2, implicit killed %d1;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000800 for (auto &MO : MI.operands()) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000801 if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)
802 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000803 }
804
805 // Can be dot new store.
806 return true;
807}
808
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000809// Can this MI to promoted to either new value store or new value jump.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000810bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000811 const SUnit *PacketSU, unsigned DepReg,
812 MachineBasicBlock::iterator &MII) {
813 if (!HII->mayBeNewStore(MI))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000814 return false;
815
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000816 // Check to see the store can be new value'ed.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000817 MachineInstr &PacketMI = *PacketSU->getInstr();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000818 if (canPromoteToNewValueStore(MI, PacketMI, DepReg))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000819 return true;
820
821 // Check to see the compare/jump can be new value'ed.
822 // This is done as a pass on its own. Don't need to check it here.
823 return false;
824}
825
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000826static bool isImplicitDependency(const MachineInstr &I, bool CheckDef,
827 unsigned DepReg) {
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +0000828 for (auto &MO : I.operands()) {
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000829 if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(DepReg))
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +0000830 return true;
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000831 if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit())
832 continue;
833 if (CheckDef == MO.isDef())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000834 return true;
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +0000835 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000836 return false;
837}
838
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000839// Check to see if an instruction can be dot new
840// There are three kinds.
841// 1. dot new on predicate - V2/V3/V4
842// 2. dot new on stores NV/ST - V4
843// 3. dot new on jump NV/J - V4 -- This is generated in a pass.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000844bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000845 const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
846 const TargetRegisterClass* RC) {
Jyotsna Vermaa46059b2013-03-28 19:44:04 +0000847 // Already a dot new instruction.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000848 if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000849 return false;
850
Krzysztof Parzyszek2a480592016-07-26 20:30:30 +0000851 if (!isNewifiable(MI, RC))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000852 return false;
853
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000854 const MachineInstr &PI = *PacketSU->getInstr();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000855
856 // The "new value" cannot come from inline asm.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000857 if (PI.isInlineAsm())
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000858 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000859
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000860 // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no
861 // sense.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000862 if (PI.isImplicitDef())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000863 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000864
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000865 // If dependency is trough an implicitly defined register, we should not
866 // newify the use.
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000867 if (isImplicitDependency(PI, true, DepReg) ||
868 isImplicitDependency(MI, false, DepReg))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000869 return false;
870
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000871 const MCInstrDesc& MCID = PI.getDesc();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000872 const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF);
Krzysztof Parzyszek55772972017-09-15 15:46:05 +0000873 if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass)
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000874 return false;
875
876 // predicate .new
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000877 if (RC == &Hexagon::PredRegsRegClass)
Krzysztof Parzyszek3cf16572017-06-01 18:02:40 +0000878 return HII->predCanBeUsedAsDotNew(PI, DepReg);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000879
880 if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))
881 return false;
882
883 // Create a dot new machine instruction to see if resources can be
884 // allocated. If not, bail out now.
885 int NewOpcode = HII->getDotNewOp(MI);
886 const MCInstrDesc &D = HII->get(NewOpcode);
887 MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc());
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +0000888 bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000889 MF.DeleteMachineInstr(NewMI);
890 if (!ResourcesAvailable)
891 return false;
892
893 // New Value Store only. New Value Jump generated as a separate pass.
894 if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII))
895 return false;
896
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000897 return true;
898}
899
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000900// Go through the packet instructions and search for an anti dependency between
901// them and DepReg from MI. Consider this case:
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000902// Trying to add
Francis Visoiu Mistriha8a83d12017-12-07 10:40:31 +0000903// a) %r1 = TFRI_cdNotPt %p3, 2
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000904// to this packet:
905// {
Francis Visoiu Mistriha8a83d12017-12-07 10:40:31 +0000906// b) %p0 = C2_or killed %p3, killed %p0
907// c) %p3 = C2_tfrrp %r23
908// d) %r1 = C2_cmovenewit %p3, 4
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000909// }
910// The P3 from a) and d) will be complements after
911// a)'s P3 is converted to .new form
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000912// Anti-dep between c) and b) is irrelevant for this case
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000913bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000914 unsigned DepReg) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +0000915 SUnit *PacketSUDep = MIToSUnit.find(&MI)->second;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000916
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000917 for (auto I : CurrentPacketMIs) {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000918 // We only care for dependencies to predicated instructions
Duncan P. N. Exon Smith6307eb52016-02-23 02:46:52 +0000919 if (!HII->isPredicated(*I))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000920 continue;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000921
922 // Scheduling Unit for current insn in the packet
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000923 SUnit *PacketSU = MIToSUnit.find(I)->second;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000924
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000925 // Look at dependencies between current members of the packet and
926 // predicate defining instruction MI. Make sure that dependency is
927 // on the exact register we care about.
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000928 if (PacketSU->isSucc(PacketSUDep)) {
929 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000930 auto &Dep = PacketSU->Succs[i];
931 if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&
932 Dep.getReg() == DepReg)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000933 return true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000934 }
935 }
936 }
937
938 return false;
939}
940
Jyotsna Verma11bd54a2013-05-14 16:36:34 +0000941/// Gets the predicate register of a predicated instruction.
Duncan P. N. Exon Smith6307eb52016-02-23 02:46:52 +0000942static unsigned getPredicatedRegister(MachineInstr &MI,
Benjamin Kramere79beac2013-05-23 15:43:11 +0000943 const HexagonInstrInfo *QII) {
Jyotsna Verma11bd54a2013-05-14 16:36:34 +0000944 /// We use the following rule: The first predicate register that is a use is
945 /// the predicate register of a predicated instruction.
Jyotsna Verma11bd54a2013-05-14 16:36:34 +0000946 assert(QII->isPredicated(MI) && "Must be predicated instruction");
947
Duncan P. N. Exon Smith6307eb52016-02-23 02:46:52 +0000948 for (auto &Op : MI.operands()) {
Jyotsna Verma11bd54a2013-05-14 16:36:34 +0000949 if (Op.isReg() && Op.getReg() && Op.isUse() &&
950 Hexagon::PredRegsRegClass.contains(Op.getReg()))
951 return Op.getReg();
952 }
953
954 llvm_unreachable("Unknown instruction operand layout");
Jyotsna Verma11bd54a2013-05-14 16:36:34 +0000955 return 0;
956}
957
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000958// Given two predicated instructions, this function detects whether
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000959// the predicates are complements.
Duncan P. N. Exon Smith6307eb52016-02-23 02:46:52 +0000960bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1,
961 MachineInstr &MI2) {
Jyotsna Verma11bd54a2013-05-14 16:36:34 +0000962 // If we don't know the predicate sense of the instructions bail out early, we
963 // need it later.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000964 if (getPredicateSense(MI1, HII) == PK_Unknown ||
965 getPredicateSense(MI2, HII) == PK_Unknown)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000966 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000967
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000968 // Scheduling unit for candidate.
Duncan P. N. Exon Smith6307eb52016-02-23 02:46:52 +0000969 SUnit *SU = MIToSUnit[&MI1];
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000970
971 // One corner case deals with the following scenario:
972 // Trying to add
Francis Visoiu Mistriha8a83d12017-12-07 10:40:31 +0000973 // a) %r24 = A2_tfrt %p0, %r25
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000974 // to this packet:
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000975 // {
Francis Visoiu Mistriha8a83d12017-12-07 10:40:31 +0000976 // b) %r25 = A2_tfrf %p0, %r24
977 // c) %p0 = C2_cmpeqi %r26, 1
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000978 // }
979 //
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000980 // On general check a) and b) are complements, but presence of c) will
981 // convert a) to .new form, and then it is not a complement.
982 // We attempt to detect it by analyzing existing dependencies in the packet.
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000983
984 // Analyze relationships between all existing members of the packet.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000985 // Look for Anti dependecy on the same predicate reg as used in the
986 // candidate.
987 for (auto I : CurrentPacketMIs) {
988 // Scheduling Unit for current insn in the packet.
989 SUnit *PacketSU = MIToSUnit.find(I)->second;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000990
991 // If this instruction in the packet is succeeded by the candidate...
992 if (PacketSU->isSucc(SU)) {
993 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000994 auto Dep = PacketSU->Succs[i];
995 // The corner case exist when there is true data dependency between
996 // candidate and one of current packet members, this dep is on
997 // predicate reg, and there already exist anti dep on the same pred in
Sirish Pandef8e5e3c2012-05-03 21:52:53 +0000998 // the packet.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +0000999 if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&
1000 Hexagon::PredRegsRegClass.contains(Dep.getReg())) {
1001 // Here I know that I is predicate setting instruction with true
1002 // data dep to candidate on the register we care about - c) in the
1003 // above example. Now I need to see if there is an anti dependency
1004 // from c) to any other instruction in the same packet on the pred
1005 // reg of interest.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001006 if (restrictingDepExistInPacket(*I, Dep.getReg()))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001007 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001008 }
1009 }
1010 }
1011 }
1012
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001013 // If the above case does not apply, check regular complement condition.
1014 // Check that the predicate register is the same and that the predicate
1015 // sense is different We also need to differentiate .old vs. .new: !p0
1016 // is not complementary to p0.new.
1017 unsigned PReg1 = getPredicatedRegister(MI1, HII);
1018 unsigned PReg2 = getPredicatedRegister(MI2, HII);
1019 return PReg1 == PReg2 &&
1020 Hexagon::PredRegsRegClass.contains(PReg1) &&
1021 Hexagon::PredRegsRegClass.contains(PReg2) &&
1022 getPredicateSense(MI1, HII) != getPredicateSense(MI2, HII) &&
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001023 HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001024}
1025
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001026// Initialize packetizer flags.
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001027void HexagonPacketizerList::initPacketizerState() {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001028 Dependence = false;
1029 PromotedToDotNew = false;
1030 GlueToNewValueJump = false;
1031 GlueAllocframeStore = false;
1032 FoundSequentialDependence = false;
Krzysztof Parzyszek8f174dd2017-10-11 15:51:44 +00001033 ChangedOffset = INT64_MAX;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001034}
1035
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001036// Ignore bundling of pseudo instructions.
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001037bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI,
1038 const MachineBasicBlock *) {
Shiva Chen801bf7e2018-05-09 02:42:00 +00001039 if (MI.isDebugInstr())
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001040 return true;
1041
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001042 if (MI.isCFIInstruction())
Krzysztof Parzyszek6bbcb312015-04-22 15:47:35 +00001043 return false;
1044
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001045 // We must print out inline assembly.
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001046 if (MI.isInlineAsm())
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001047 return false;
1048
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001049 if (MI.isImplicitDef())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001050 return false;
1051
1052 // We check if MI has any functional units mapped to it. If it doesn't,
1053 // we ignore the instruction.
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001054 const MCInstrDesc& TID = MI.getDesc();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001055 auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass());
Hal Finkel8db55472012-06-22 20:27:13 +00001056 unsigned FuncUnits = IS->getUnits();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001057 return !FuncUnits;
1058}
1059
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001060bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) {
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001061 // Ensure any bundles created by gather packetize remain seperate.
1062 if (MI.isBundle())
1063 return true;
1064
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001065 if (MI.isEHLabel() || MI.isCFIInstruction())
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001066 return true;
1067
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001068 // Consider inline asm to not be a solo instruction by default.
1069 // Inline asm will be put in a packet temporarily, but then it will be
1070 // removed, and placed outside of the packet (before or after, depending
1071 // on dependencies). This is to reduce the impact of inline asm as a
1072 // "packet splitting" instruction.
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001073 if (MI.isInlineAsm() && !ScheduleInlineAsm)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001074 return true;
1075
1076 // From Hexagon V4 Programmer's Reference Manual 3.4.4 Grouping constraints:
1077 // trap, pause, barrier, icinva, isync, and syncht are solo instructions.
1078 // They must not be grouped with other instructions in a packet.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001079 if (isSchedBarrier(MI))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001080 return true;
1081
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001082 if (HII->isSolo(MI))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001083 return true;
1084
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001085 if (MI.getOpcode() == Hexagon::A2_nop)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001086 return true;
1087
1088 return false;
1089}
1090
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001091// Quick check if instructions MI and MJ cannot coexist in the same packet.
1092// Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm",
1093// but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
1094// For full test call this function twice:
1095// cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI)
1096// Doing the test only one way saves the amount of code in this function,
1097// since every test would need to be repeated with the MI and MJ reversed.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001098static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ,
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001099 const HexagonInstrInfo &HII) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001100 const MachineFunction *MF = MI.getParent()->getParent();
Krzysztof Parzyszekd8b780d2018-06-20 13:56:09 +00001101 if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() &&
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001102 HII.isHVXMemWithAIndirect(MI, MJ))
1103 return true;
1104
1105 // An inline asm cannot be together with a branch, because we may not be
1106 // able to remove the asm out after packetizing (i.e. if the asm must be
1107 // moved past the bundle). Similarly, two asms cannot be together to avoid
1108 // complications when determining their relative order outside of a bundle.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001109 if (MI.isInlineAsm())
1110 return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() ||
1111 MJ.isCall() || MJ.isTerminator();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001112
Krzysztof Parzyszek639545b2016-08-19 16:57:05 +00001113 switch (MI.getOpcode()) {
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001114 case Hexagon::S2_storew_locked:
1115 case Hexagon::S4_stored_locked:
1116 case Hexagon::L2_loadw_locked:
1117 case Hexagon::L4_loadd_locked:
Krzysztof Parzyszek5c2944c2018-06-19 17:26:20 +00001118 case Hexagon::Y2_dccleana:
1119 case Hexagon::Y2_dccleaninva:
1120 case Hexagon::Y2_dcinva:
1121 case Hexagon::Y2_dczeroa:
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001122 case Hexagon::Y4_l2fetch:
1123 case Hexagon::Y5_l2fetch: {
Krzysztof Parzyszek639545b2016-08-19 16:57:05 +00001124 // These instructions can only be grouped with ALU32 or non-floating-point
1125 // XTYPE instructions. Since there is no convenient way of identifying fp
1126 // XTYPE instructions, only allow grouping with ALU32 for now.
1127 unsigned TJ = HII.getType(MJ);
Krzysztof Parzyszek5ea971c2017-02-07 17:47:37 +00001128 if (TJ != HexagonII::TypeALU32_2op &&
1129 TJ != HexagonII::TypeALU32_3op &&
1130 TJ != HexagonII::TypeALU32_ADDI)
Krzysztof Parzyszek639545b2016-08-19 16:57:05 +00001131 return true;
1132 break;
1133 }
1134 default:
1135 break;
1136 }
1137
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001138 // "False" really means that the quick check failed to determine if
1139 // I and J cannot coexist.
1140 return false;
1141}
1142
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001143// Full, symmetric check.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001144bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI,
1145 const MachineInstr &MJ) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001146 return cannotCoexistAsymm(MI, MJ, *HII) || cannotCoexistAsymm(MJ, MI, *HII);
1147}
1148
1149void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) {
1150 for (auto &B : MF) {
1151 MachineBasicBlock::iterator BundleIt;
1152 MachineBasicBlock::instr_iterator NextI;
1153 for (auto I = B.instr_begin(), E = B.instr_end(); I != E; I = NextI) {
1154 NextI = std::next(I);
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001155 MachineInstr &MI = *I;
1156 if (MI.isBundle())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001157 BundleIt = I;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001158 if (!MI.isInsideBundle())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001159 continue;
1160
1161 // Decide on where to insert the instruction that we are pulling out.
1162 // Debug instructions always go before the bundle, but the placement of
1163 // INLINE_ASM depends on potential dependencies. By default, try to
1164 // put it before the bundle, but if the asm writes to a register that
1165 // other instructions in the bundle read, then we need to place it
1166 // after the bundle (to preserve the bundle semantics).
1167 bool InsertBeforeBundle;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001168 if (MI.isInlineAsm())
1169 InsertBeforeBundle = !hasWriteToReadDep(MI, *BundleIt, HRI);
1170 else if (MI.isDebugValue())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001171 InsertBeforeBundle = true;
1172 else
1173 continue;
1174
1175 BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle);
1176 }
1177 }
1178}
1179
1180// Check if a given instruction is of class "system".
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001181static bool isSystemInstr(const MachineInstr &MI) {
1182 unsigned Opc = MI.getOpcode();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001183 switch (Opc) {
1184 case Hexagon::Y2_barrier:
1185 case Hexagon::Y2_dcfetchbo:
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001186 case Hexagon::Y4_l2fetch:
1187 case Hexagon::Y5_l2fetch:
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001188 return true;
1189 }
1190 return false;
1191}
1192
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001193bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I,
1194 const MachineInstr &J) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001195 // The dependence graph may not include edges between dead definitions,
1196 // so without extra checks, we could end up packetizing two instruction
1197 // defining the same (dead) register.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001198 if (I.isCall() || J.isCall())
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001199 return false;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001200 if (HII->isPredicated(I) || HII->isPredicated(J))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001201 return false;
1202
1203 BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001204 for (auto &MO : I.operands()) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001205 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1206 continue;
1207 DeadDefs[MO.getReg()] = true;
1208 }
1209
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001210 for (auto &MO : J.operands()) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001211 if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1212 continue;
1213 unsigned R = MO.getReg();
1214 if (R != Hexagon::USR_OVF && DeadDefs[R])
1215 return true;
1216 }
1217 return false;
1218}
1219
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001220bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I,
1221 const MachineInstr &J) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001222 // A save callee-save register function call can only be in a packet
1223 // with instructions that don't write to the callee-save registers.
1224 if ((HII->isSaveCalleeSavedRegsCall(I) &&
1225 doesModifyCalleeSavedReg(J, HRI)) ||
1226 (HII->isSaveCalleeSavedRegsCall(J) &&
1227 doesModifyCalleeSavedReg(I, HRI)))
1228 return true;
1229
1230 // Two control flow instructions cannot go in the same packet.
1231 if (isControlFlow(I) && isControlFlow(J))
1232 return true;
1233
1234 // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot
1235 // contain a speculative indirect jump,
1236 // a new-value compare jump or a dealloc_return.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001237 auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {
1238 if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001239 return true;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001240 if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001241 return true;
1242 return false;
1243 };
1244
1245 if (HII->isLoopN(I) && isBadForLoopN(J))
1246 return true;
1247 if (HII->isLoopN(J) && isBadForLoopN(I))
1248 return true;
1249
1250 // dealloc_return cannot appear in the same packet as a conditional or
1251 // unconditional jump.
1252 return HII->isDeallocRet(I) &&
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001253 (J.isBranch() || J.isCall() || J.isBarrier());
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001254}
1255
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +00001256bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I,
1257 const MachineInstr &J) {
1258 // Adding I to a packet that has J.
1259
1260 // Regmasks are not reflected in the scheduling dependency graph, so
1261 // we need to check them manually. This code assumes that regmasks only
1262 // occur on calls, and the problematic case is when we add an instruction
1263 // defining a register R to a packet that has a call that clobbers R via
1264 // a regmask. Those cannot be packetized together, because the call will
1265 // be executed last. That's also a reson why it is ok to add a call
1266 // clobbering R to a packet that defines R.
1267
1268 // Look for regmasks in J.
1269 for (const MachineOperand &OpJ : J.operands()) {
1270 if (!OpJ.isRegMask())
1271 continue;
1272 assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call");
1273 for (const MachineOperand &OpI : I.operands()) {
1274 if (OpI.isReg()) {
1275 if (OpJ.clobbersPhysReg(OpI.getReg()))
1276 return true;
1277 } else if (OpI.isRegMask()) {
1278 // Both are regmasks. Assume that they intersect.
1279 return true;
1280 }
1281 }
1282 }
1283 return false;
1284}
1285
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001286bool HexagonPacketizerList::hasV4SpecificDependence(const MachineInstr &I,
1287 const MachineInstr &J) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001288 bool SysI = isSystemInstr(I), SysJ = isSystemInstr(J);
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001289 bool StoreI = I.mayStore(), StoreJ = J.mayStore();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001290 if ((SysI && StoreJ) || (SysJ && StoreI))
1291 return true;
1292
1293 if (StoreI && StoreJ) {
1294 if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I))
1295 return true;
1296 } else {
1297 // A memop cannot be in the same packet with another memop or a store.
1298 // Two stores can be together, but here I and J cannot both be stores.
1299 bool MopStI = HII->isMemOp(I) || StoreI;
1300 bool MopStJ = HII->isMemOp(J) || StoreJ;
1301 if (MopStI && MopStJ)
1302 return true;
1303 }
1304
1305 return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J));
1306}
1307
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001308// SUI is the current instruction that is out side of the current packet.
1309// SUJ is the current instruction inside the current packet against which that
1310// SUI will be packetized.
1311bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001312 assert(SUI->getInstr() && SUJ->getInstr());
1313 MachineInstr &I = *SUI->getInstr();
1314 MachineInstr &J = *SUJ->getInstr();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001315
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001316 // Clear IgnoreDepMIs when Packet starts.
1317 if (CurrentPacketMIs.size() == 1)
1318 IgnoreDepMIs.clear();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001319
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001320 MachineBasicBlock::iterator II = I.getIterator();
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001321
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001322 // Solo instructions cannot go in the packet.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001323 assert(!isSoloInstruction(I) && "Unexpected solo instr!");
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001324
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001325 if (cannotCoexist(I, J))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001326 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001327
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001328 Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J);
1329 if (Dependence)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001330 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001331
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +00001332 // Regmasks are not accounted for in the scheduling graph, so we need
1333 // to explicitly check for dependencies caused by them. They should only
1334 // appear on calls, so it's not too pessimistic to reject all regmask
1335 // dependencies.
1336 Dependence = hasRegMaskDependence(I, J);
1337 if (Dependence)
1338 return false;
1339
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001340 // V4 allows dual stores. It does not allow second store, if the first
1341 // store is not in SLOT0. New value store, new value jump, dealloc_return
1342 // and memop always take SLOT0. Arch spec 3.4.4.2.
1343 Dependence = hasV4SpecificDependence(I, J);
1344 if (Dependence)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001345 return false;
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001346
1347 // If an instruction feeds new value jump, glue it.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001348 MachineBasicBlock::iterator NextMII = I.getIterator();
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001349 ++NextMII;
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001350 if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) {
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +00001351 MachineInstr &NextMI = *NextMII;
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001352
1353 bool secondRegMatch = false;
Duncan P. N. Exon Smith98226e32016-07-12 01:55:32 +00001354 const MachineOperand &NOp0 = NextMI.getOperand(0);
1355 const MachineOperand &NOp1 = NextMI.getOperand(1);
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001356
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001357 if (NOp1.isReg() && I.getOperand(0).getReg() == NOp1.getReg())
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001358 secondRegMatch = true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001359
Krzysztof Parzyszekc4a9a8d2017-10-11 21:20:43 +00001360 for (MachineInstr *PI : CurrentPacketMIs) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001361 // NVJ can not be part of the dual jump - Arch Spec: section 7.8.
Krzysztof Parzyszekc4a9a8d2017-10-11 21:20:43 +00001362 if (PI->isCall()) {
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001363 Dependence = true;
1364 break;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001365 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001366 // Validate:
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001367 // 1. Packet does not have a store in it.
1368 // 2. If the first operand of the nvj is newified, and the second
1369 // operand is also a reg, it (second reg) is not defined in
1370 // the same packet.
1371 // 3. If the second operand of the nvj is newified, (which means
1372 // first operand is also a reg), first reg is not defined in
1373 // the same packet.
Krzysztof Parzyszekc4a9a8d2017-10-11 21:20:43 +00001374 if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||
1375 HII->isLoopN(*PI)) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001376 Dependence = true;
1377 break;
1378 }
1379 // Check #2/#3.
1380 const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;
Krzysztof Parzyszekc4a9a8d2017-10-11 21:20:43 +00001381 if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) {
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001382 Dependence = true;
1383 break;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001384 }
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001385 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001386
Krzysztof Parzyszekc4a9a8d2017-10-11 21:20:43 +00001387 GlueToNewValueJump = true;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001388 if (Dependence)
Colin LeMahieu4fd203d2015-02-09 21:56:37 +00001389 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001390 }
1391
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001392 // There no dependency between a prolog instruction and its successor.
1393 if (!SUJ->isSucc(SUI))
1394 return true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001395
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001396 for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {
1397 if (FoundSequentialDependence)
1398 break;
1399
1400 if (SUJ->Succs[i].getSUnit() != SUI)
1401 continue;
1402
1403 SDep::Kind DepType = SUJ->Succs[i].getKind();
1404 // For direct calls:
1405 // Ignore register dependences for call instructions for packetization
1406 // purposes except for those due to r31 and predicate registers.
1407 //
1408 // For indirect calls:
1409 // Same as direct calls + check for true dependences to the register
1410 // used in the indirect call.
1411 //
1412 // We completely ignore Order dependences for call instructions.
1413 //
1414 // For returns:
1415 // Ignore register dependences for return instructions like jumpr,
1416 // dealloc return unless we have dependencies on the explicit uses
1417 // of the registers used by jumpr (like r31) or dealloc return
1418 // (like r29 or r30).
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001419 unsigned DepReg = 0;
1420 const TargetRegisterClass *RC = nullptr;
1421 if (DepType == SDep::Data) {
1422 DepReg = SUJ->Succs[i].getReg();
1423 RC = HRI->getMinimalPhysRegClass(DepReg);
1424 }
1425
Krzysztof Parzyszek38e2ccc2016-08-23 16:01:01 +00001426 if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001427 if (!isRegDependence(DepType))
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001428 continue;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001429 if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg()))
1430 continue;
1431 }
1432
1433 if (DepType == SDep::Data) {
1434 if (canPromoteToDotCur(J, SUJ, DepReg, II, RC))
1435 if (promoteToDotCur(J, DepType, II, RC))
1436 continue;
1437 }
1438
1439 // Data dpendence ok if we have load.cur.
1440 if (DepType == SDep::Data && HII->isDotCurInst(J)) {
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001441 if (HII->isHVXVec(I))
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001442 continue;
1443 }
1444
1445 // For instructions that can be promoted to dot-new, try to promote.
1446 if (DepType == SDep::Data) {
1447 if (canPromoteToDotNew(I, SUJ, DepReg, II, RC)) {
1448 if (promoteToDotNew(I, DepType, II, RC)) {
1449 PromotedToDotNew = true;
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001450 if (cannotCoexist(I, J))
1451 FoundSequentialDependence = true;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001452 continue;
1453 }
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001454 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001455 if (HII->isNewValueJump(I))
1456 continue;
1457 }
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001458
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001459 // For predicated instructions, if the predicates are complements then
1460 // there can be no dependence.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001461 if (HII->isPredicated(I) && HII->isPredicated(J) &&
1462 arePredicatesComplements(I, J)) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001463 // Not always safe to do this translation.
1464 // DAG Builder attempts to reduce dependence edges using transitive
1465 // nature of dependencies. Here is an example:
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001466 //
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001467 // r0 = tfr_pt ... (1)
1468 // r0 = tfr_pf ... (2)
1469 // r0 = tfr_pt ... (3)
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001470 //
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001471 // There will be an output dependence between (1)->(2) and (2)->(3).
1472 // However, there is no dependence edge between (1)->(3). This results
1473 // in all 3 instructions going in the same packet. We ignore dependce
1474 // only once to avoid this situation.
David Majnemer0d955d02016-08-11 22:21:41 +00001475 auto Itr = find(IgnoreDepMIs, &J);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001476 if (Itr != IgnoreDepMIs.end()) {
1477 Dependence = true;
1478 return false;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001479 }
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001480 IgnoreDepMIs.push_back(&I);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001481 continue;
1482 }
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001483
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001484 // Ignore Order dependences between unconditional direct branches
1485 // and non-control-flow instructions.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001486 if (isDirectJump(I) && !J.isBranch() && !J.isCall() &&
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001487 DepType == SDep::Order)
1488 continue;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001489
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001490 // Ignore all dependences for jumps except for true and output
1491 // dependences.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001492 if (I.isConditionalBranch() && DepType != SDep::Data &&
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001493 DepType != SDep::Output)
1494 continue;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001495
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001496 if (DepType == SDep::Output) {
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001497 FoundSequentialDependence = true;
1498 break;
1499 }
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001500
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001501 // For Order dependences:
1502 // 1. On V4 or later, volatile loads/stores can be packetized together,
1503 // unless other rules prevent is.
1504 // 2. Store followed by a load is not allowed.
1505 // 3. Store followed by a store is only valid on V4 or later.
1506 // 4. Load followed by any memory operation is allowed.
1507 if (DepType == SDep::Order) {
1508 if (!PacketizeVolatiles) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001509 bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001510 if (OrdRefs) {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001511 FoundSequentialDependence = true;
1512 break;
1513 }
1514 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001515 // J is first, I is second.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001516 bool LoadJ = J.mayLoad(), StoreJ = J.mayStore();
1517 bool LoadI = I.mayLoad(), StoreI = I.mayStore();
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001518 bool NVStoreJ = HII->isNewValueStore(J);
1519 bool NVStoreI = HII->isNewValueStore(I);
1520 bool IsVecJ = HII->isHVXVec(J);
1521 bool IsVecI = HII->isHVXVec(I);
1522
Krzysztof Parzyszekd8b780d2018-06-20 13:56:09 +00001523 if (Slot1Store && MF.getSubtarget<HexagonSubtarget>().hasV65Ops() &&
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001524 ((LoadJ && StoreI && !NVStoreI) ||
1525 (StoreJ && LoadI && !NVStoreJ)) &&
1526 (J.getOpcode() != Hexagon::S2_allocframe &&
1527 I.getOpcode() != Hexagon::S2_allocframe) &&
1528 (J.getOpcode() != Hexagon::L2_deallocframe &&
1529 I.getOpcode() != Hexagon::L2_deallocframe) &&
1530 (!HII->isMemOp(J) && !HII->isMemOp(I)) && (!IsVecJ && !IsVecI))
1531 setmemShufDisabled(true);
1532 else
1533 if (StoreJ && LoadI && alias(J, I)) {
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001534 FoundSequentialDependence = true;
1535 break;
1536 }
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001537
1538 if (!StoreJ)
1539 if (!LoadJ || (!LoadI && !StoreI)) {
1540 // If J is neither load nor store, assume a dependency.
1541 // If J is a load, but I is neither, also assume a dependency.
1542 FoundSequentialDependence = true;
1543 break;
1544 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001545 // Store followed by store: not OK on V2.
1546 // Store followed by load: not OK on all.
1547 // Load followed by store: OK on all.
1548 // Load followed by load: OK on all.
1549 continue;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001550 }
1551
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001552 // For V4, special case ALLOCFRAME. Even though there is dependency
1553 // between ALLOCFRAME and subsequent store, allow it to be packetized
1554 // in a same packet. This implies that the store is using the caller's
1555 // SP. Hence, offset needs to be updated accordingly.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001556 if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) {
1557 unsigned Opc = I.getOpcode();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001558 switch (Opc) {
1559 case Hexagon::S2_storerd_io:
1560 case Hexagon::S2_storeri_io:
1561 case Hexagon::S2_storerh_io:
1562 case Hexagon::S2_storerb_io:
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001563 if (I.getOperand(0).getReg() == HRI->getStackRegister()) {
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +00001564 // Since this store is to be glued with allocframe in the same
1565 // packet, it will use SP of the previous stack frame, i.e.
1566 // caller's SP. Therefore, we need to recalculate offset
1567 // according to this change.
1568 GlueAllocframeStore = useCallersSP(I);
1569 if (GlueAllocframeStore)
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001570 continue;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001571 }
1572 default:
1573 break;
1574 }
1575 }
1576
Krzysztof Parzyszekadb7ff02016-05-06 19:13:38 +00001577 // There are certain anti-dependencies that cannot be ignored.
1578 // Specifically:
Francis Visoiu Mistriha8a83d12017-12-07 10:40:31 +00001579 // J2_call ... implicit-def %r0 ; SUJ
Krzysztof Parzyszekadb7ff02016-05-06 19:13:38 +00001580 // R0 = ... ; SUI
1581 // Those cannot be packetized together, since the call will observe
1582 // the effect of the assignment to R0.
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +00001583 if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) {
Krzysztof Parzyszekadb7ff02016-05-06 19:13:38 +00001584 // Check if I defines any volatile register. We should also check
1585 // registers that the call may read, but these happen to be a
1586 // subset of the volatile register set.
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +00001587 for (const MachineOperand &Op : I.operands()) {
1588 if (Op.isReg() && Op.isDef()) {
1589 unsigned R = Op.getReg();
1590 if (!J.readsRegister(R, HRI) && !J.modifiesRegister(R, HRI))
1591 continue;
1592 } else if (!Op.isRegMask()) {
1593 // If I has a regmask assume dependency.
Krzysztof Parzyszekadb7ff02016-05-06 19:13:38 +00001594 continue;
Krzysztof Parzyszek1aaf41a2017-02-17 22:14:51 +00001595 }
Krzysztof Parzyszekadb7ff02016-05-06 19:13:38 +00001596 FoundSequentialDependence = true;
1597 break;
1598 }
1599 }
1600
1601 // Skip over remaining anti-dependences. Two instructions that are
1602 // anti-dependent can share a packet, since in most such cases all
1603 // operands are read before any modifications take place.
1604 // The exceptions are branch and call instructions, since they are
1605 // executed after all other instructions have completed (at least
1606 // conceptually).
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001607 if (DepType != SDep::Anti) {
1608 FoundSequentialDependence = true;
1609 break;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001610 }
1611 }
1612
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001613 if (FoundSequentialDependence) {
1614 Dependence = true;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001615 return false;
1616 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001617
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001618 return true;
1619}
1620
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001621bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001622 assert(SUI->getInstr() && SUJ->getInstr());
1623 MachineInstr &I = *SUI->getInstr();
1624 MachineInstr &J = *SUJ->getInstr();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001625
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001626 bool Coexist = !cannotCoexist(I, J);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001627
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001628 if (Coexist && !Dependence)
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001629 return true;
1630
1631 // Check if the instruction was promoted to a dot-new. If so, demote it
1632 // back into a dot-old.
1633 if (PromotedToDotNew)
1634 demoteToDotOld(I);
1635
1636 cleanUpDotCur();
1637 // Check if the instruction (must be a store) was glued with an allocframe
1638 // instruction. If so, restore its offset to its original value, i.e. use
1639 // current SP instead of caller's SP.
1640 if (GlueAllocframeStore) {
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +00001641 useCalleesSP(I);
1642 GlueAllocframeStore = false;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001643 }
Krzysztof Parzyszek8f174dd2017-10-11 15:51:44 +00001644
1645 if (ChangedOffset != INT64_MAX)
1646 undoChangedOffset(I);
Krzysztof Parzyszekc4a9a8d2017-10-11 21:20:43 +00001647
1648 if (GlueToNewValueJump) {
1649 // Putting I and J together would prevent the new-value jump from being
1650 // packetized with the producer. In that case I and J must be separated.
1651 GlueToNewValueJump = false;
1652 return false;
1653 }
1654
1655 if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) {
Krzysztof Parzyszek8f174dd2017-10-11 15:51:44 +00001656 FoundSequentialDependence = false;
1657 Dependence = false;
1658 return true;
1659 }
1660
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001661 return false;
1662}
1663
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001664
1665bool HexagonPacketizerList::foundLSInPacket() {
1666 bool FoundLoad = false;
1667 bool FoundStore = false;
1668
1669 for (auto MJ : CurrentPacketMIs) {
1670 unsigned Opc = MJ->getOpcode();
1671 if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe)
1672 continue;
1673 if (HII->isMemOp(*MJ))
1674 continue;
1675 if (MJ->mayLoad())
1676 FoundLoad = true;
1677 if (MJ->mayStore() && !HII->isNewValueStore(*MJ))
1678 FoundStore = true;
1679 }
1680 return FoundLoad && FoundStore;
1681}
1682
1683
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001684MachineBasicBlock::iterator
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001685HexagonPacketizerList::addToPacket(MachineInstr &MI) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001686 MachineBasicBlock::iterator MII = MI.getIterator();
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001687 MachineBasicBlock *MBB = MI.getParent();
Krzysztof Parzyszek9aaf9232017-05-02 18:12:19 +00001688
Eugene Zelenko3b873362017-09-28 22:27:31 +00001689 if (CurrentPacketMIs.empty())
Krzysztof Parzyszek9aaf9232017-05-02 18:12:19 +00001690 PacketStalls = false;
1691 PacketStalls |= producesStall(MI);
1692
Krzysztof Parzyszekb7e54e82018-04-06 18:19:22 +00001693 if (MI.isImplicitDef()) {
1694 // Add to the packet to allow subsequent instructions to be checked
1695 // properly.
1696 CurrentPacketMIs.push_back(&MI);
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001697 return MII;
Krzysztof Parzyszekb7e54e82018-04-06 18:19:22 +00001698 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001699 assert(ResourceTracker->canReserveResources(MI));
1700
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001701 bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001702 bool Good = true;
1703
1704 if (GlueToNewValueJump) {
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001705 MachineInstr &NvjMI = *++MII;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001706 // We need to put both instructions in the same packet: MI and NvjMI.
1707 // Either of them can require a constant extender. Try to add both to
1708 // the current packet, and if that fails, end the packet and start a
1709 // new one.
1710 ResourceTracker->reserveResources(MI);
1711 if (ExtMI)
1712 Good = tryAllocateResourcesForConstExt(true);
1713
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001714 bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001715 if (Good) {
1716 if (ResourceTracker->canReserveResources(NvjMI))
1717 ResourceTracker->reserveResources(NvjMI);
1718 else
1719 Good = false;
1720 }
1721 if (Good && ExtNvjMI)
1722 Good = tryAllocateResourcesForConstExt(true);
1723
1724 if (!Good) {
1725 endPacket(MBB, MI);
1726 assert(ResourceTracker->canReserveResources(MI));
1727 ResourceTracker->reserveResources(MI);
1728 if (ExtMI) {
1729 assert(canReserveResourcesForConstExt());
1730 tryAllocateResourcesForConstExt(true);
1731 }
1732 assert(ResourceTracker->canReserveResources(NvjMI));
1733 ResourceTracker->reserveResources(NvjMI);
1734 if (ExtNvjMI) {
1735 assert(canReserveResourcesForConstExt());
1736 reserveResourcesForConstExt();
1737 }
1738 }
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001739 CurrentPacketMIs.push_back(&MI);
1740 CurrentPacketMIs.push_back(&NvjMI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001741 return MII;
1742 }
1743
1744 ResourceTracker->reserveResources(MI);
1745 if (ExtMI && !tryAllocateResourcesForConstExt(true)) {
1746 endPacket(MBB, MI);
1747 if (PromotedToDotNew)
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001748 demoteToDotOld(MI);
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +00001749 if (GlueAllocframeStore) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001750 useCalleesSP(MI);
Krzysztof Parzyszek3b4682f2016-07-26 14:24:46 +00001751 GlueAllocframeStore = false;
1752 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001753 ResourceTracker->reserveResources(MI);
1754 reserveResourcesForConstExt();
1755 }
1756
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001757 CurrentPacketMIs.push_back(&MI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001758 return MII;
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001759}
1760
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001761void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB,
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001762 MachineBasicBlock::iterator MI) {
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001763 // Replace VLIWPacketizerList::endPacket(MBB, MI).
1764
1765 bool memShufDisabled = getmemShufDisabled();
1766 if (memShufDisabled && !foundLSInPacket()) {
1767 setmemShufDisabled(false);
Nicola Zaghend34e60c2018-05-14 12:53:11 +00001768 LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n");
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001769 }
1770 memShufDisabled = getmemShufDisabled();
1771
1772 if (CurrentPacketMIs.size() > 1) {
1773 MachineBasicBlock::instr_iterator FirstMI(CurrentPacketMIs.front());
1774 MachineBasicBlock::instr_iterator LastMI(MI.getInstrIterator());
1775 finalizeBundle(*MBB, FirstMI, LastMI);
1776
1777 auto BundleMII = std::prev(FirstMI);
1778 if (memShufDisabled)
1779 HII->setBundleNoShuf(BundleMII);
1780
1781 setmemShufDisabled(false);
1782 }
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001783 OldPacketMIs = CurrentPacketMIs;
Krzysztof Parzyszeka8ab1b72017-12-11 18:57:54 +00001784 CurrentPacketMIs.clear();
1785
1786 ResourceTracker->clearResources();
Nicola Zaghend34e60c2018-05-14 12:53:11 +00001787 LLVM_DEBUG(dbgs() << "End packet\n");
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001788}
1789
Duncan P. N. Exon Smith57022872016-02-27 19:09:00 +00001790bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) {
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001791 return !producesStall(MI);
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001792}
1793
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001794// V60 forward scheduling.
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001795bool HexagonPacketizerList::producesStall(const MachineInstr &I) {
Krzysztof Parzyszek9aaf9232017-05-02 18:12:19 +00001796 // If the packet already stalls, then ignore the stall from a subsequent
1797 // instruction in the same packet.
1798 if (PacketStalls)
1799 return false;
1800
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001801 // Check whether the previous packet is in a different loop. If this is the
1802 // case, there is little point in trying to avoid a stall because that would
1803 // favor the rare case (loop entry) over the common case (loop iteration).
1804 //
1805 // TODO: We should really be able to check all the incoming edges if this is
1806 // the first packet in a basic block, so we can avoid stalls from the loop
1807 // backedge.
1808 if (!OldPacketMIs.empty()) {
1809 auto *OldBB = OldPacketMIs.front()->getParent();
Krzysztof Parzyszekf0b34a52016-07-29 21:49:42 +00001810 auto *ThisBB = I.getParent();
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001811 if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB))
1812 return false;
1813 }
1814
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001815 SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)];
Krzysztof Parzyszek9aaf9232017-05-02 18:12:19 +00001816
Krzysztof Parzyszekaca8f322018-04-06 18:13:11 +00001817 // If the latency is 0 and there is a data dependence between this
1818 // instruction and any instruction in the current packet, we disregard any
1819 // potential stalls due to the instructions in the previous packet. Most of
1820 // the instruction pairs that can go together in the same packet have 0
1821 // latency between them. The exceptions are
1822 // 1. NewValueJumps as they're generated much later and the latencies can't
1823 // be changed at that point.
1824 // 2. .cur instructions, if its consumer has a 0 latency successor (such as
1825 // .new). In this case, the latency between .cur and the consumer stays
1826 // non-zero even though we can have both .cur and .new in the same packet.
1827 // Changing the latency to 0 is not an option as it causes software pipeliner
1828 // to not pipeline in some cases.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001829
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001830 // For Example:
1831 // {
1832 // I1: v6.cur = vmem(r0++#1)
1833 // I2: v7 = valign(v6,v4,r2)
1834 // I3: vmem(r5++#1) = v7.new
1835 // }
1836 // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2.
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001837
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001838 for (auto J : CurrentPacketMIs) {
1839 SUnit *SUJ = MIToSUnit[J];
1840 for (auto &Pred : SUI->Preds)
Krzysztof Parzyszekaca8f322018-04-06 18:13:11 +00001841 if (Pred.getSUnit() == SUJ)
1842 if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) ||
1843 HII->isNewValueJump(I) || HII->isToBeScheduledASAP(*J, I))
1844 return false;
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001845 }
1846
Krzysztof Parzyszek9aaf9232017-05-02 18:12:19 +00001847 // Check if the latency is greater than one between this instruction and any
1848 // instruction in the previous packet.
Krzysztof Parzyszek2af50372017-05-03 20:10:36 +00001849 for (auto J : OldPacketMIs) {
1850 SUnit *SUJ = MIToSUnit[J];
1851 for (auto &Pred : SUI->Preds)
1852 if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1)
1853 return true;
1854 }
1855
Krzysztof Parzyszek56bbf542015-12-16 19:36:12 +00001856 return false;
1857}
1858
Sirish Pandef8e5e3c2012-05-03 21:52:53 +00001859//===----------------------------------------------------------------------===//
1860// Public Constructor Functions
1861//===----------------------------------------------------------------------===//
1862
1863FunctionPass *llvm::createHexagonPacketizer() {
1864 return new HexagonPacketizer();
1865}