Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 8f66df92ba28f7db8c63bee46780b825750ad8d7 / . / llvm / lib / Target / AArch64 / AArch64LoadStoreOptimizer.cpp
blob: a2e0376c50bb5bad477d91d4bc8a02b97ce700af [file] [log] [blame]
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1//=- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -*- C++ -*-=//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains a pass that performs load / store related peephole
11// optimizations. This pass should be run after register allocation.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AArch64InstrInfo.h"
Eric Christopherd9134482014-08-04 21:25:23 +0000[diff] [blame]16#include "AArch64Subtarget.h"
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]17#include "MCTargetDesc/AArch64AddressingModes.h"
18#include "llvm/ADT/BitVector.h"
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]19#include "llvm/ADT/SmallVector.h"
Benjamin Kramer1f8930e2014-07-25 11:42:14 +0000[diff] [blame]20#include "llvm/ADT/Statistic.h"
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]21#include "llvm/CodeGen/MachineBasicBlock.h"
22#include "llvm/CodeGen/MachineFunctionPass.h"
23#include "llvm/CodeGen/MachineInstr.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/Support/ErrorHandling.h"
28#include "llvm/Support/raw_ostream.h"
Benjamin Kramer1f8930e2014-07-25 11:42:14 +0000[diff] [blame]29#include "llvm/Target/TargetInstrInfo.h"
30#include "llvm/Target/TargetMachine.h"
31#include "llvm/Target/TargetRegisterInfo.h"
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]32using namespace llvm;
33
34#define DEBUG_TYPE "aarch64-ldst-opt"
35
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]36STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
37STATISTIC(NumPostFolded, "Number of post-index updates folded");
38STATISTIC(NumPreFolded, "Number of pre-index updates folded");
39STATISTIC(NumUnscaledPairCreated,
40 "Number of load/store from unscaled generated");
Jun Bum Limc12c2792015-11-19 18:41:27 +0000[diff] [blame]41STATISTIC(NumNarrowLoadsPromoted, "Number of narrow loads promoted");
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]42STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]43STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]44
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]45// The LdStLimit limits how far we search for load/store pairs.
46static cl::opt<unsigned> LdStLimit("aarch64-load-store-scan-limit",
Tilmann Scheller5d8d72c2014-06-04 12:40:35 +0000[diff] [blame]47 cl::init(20), cl::Hidden);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]48
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]49// The UpdateLimit limits how far we search for update instructions when we form
50// pre-/post-index instructions.
51static cl::opt<unsigned> UpdateLimit("aarch64-update-scan-limit", cl::init(100),
52 cl::Hidden);
53
Jun Bum Lim33be4992016-05-06 15:08:57 +0000[diff] [blame]54static cl::opt<bool> EnableNarrowLdMerge("enable-narrow-ld-merge", cl::Hidden,
Jun Bum Limb21d4e12016-05-20 18:45:49 +0000[diff] [blame]55 cl::init(false),
Jun Bum Lim33be4992016-05-06 15:08:57 +0000[diff] [blame]56 cl::desc("Enable narrow load merge"));
57
Chad Rosier96530b32015-08-05 13:44:51 +0000[diff] [blame]58#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
59
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]60namespace {
Chad Rosier96a18a92015-07-21 17:42:04 +0000[diff] [blame]61
62typedef struct LdStPairFlags {
63 // If a matching instruction is found, MergeForward is set to true if the
64 // merge is to remove the first instruction and replace the second with
65 // a pair-wise insn, and false if the reverse is true.
66 bool MergeForward;
67
68 // SExtIdx gives the index of the result of the load pair that must be
69 // extended. The value of SExtIdx assumes that the paired load produces the
70 // value in this order: (I, returned iterator), i.e., -1 means no value has
71 // to be extended, 0 means I, and 1 means the returned iterator.
72 int SExtIdx;
73
74 LdStPairFlags() : MergeForward(false), SExtIdx(-1) {}
75
76 void setMergeForward(bool V = true) { MergeForward = V; }
77 bool getMergeForward() const { return MergeForward; }
78
79 void setSExtIdx(int V) { SExtIdx = V; }
80 int getSExtIdx() const { return SExtIdx; }
81
82} LdStPairFlags;
83
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]84struct AArch64LoadStoreOpt : public MachineFunctionPass {
85 static char ID;
Jun Bum Lim22fe15e2015-11-06 16:27:47 +0000[diff] [blame]86 AArch64LoadStoreOpt() : MachineFunctionPass(ID) {
Chad Rosier96530b32015-08-05 13:44:51 +0000[diff] [blame]87 initializeAArch64LoadStoreOptPass(*PassRegistry::getPassRegistry());
88 }
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]89
90 const AArch64InstrInfo *TII;
91 const TargetRegisterInfo *TRI;
Oliver Stannardd414c992015-11-10 11:04:18 +0000[diff] [blame]92 const AArch64Subtarget *Subtarget;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]93
Chad Rosierbba881e2016-02-02 15:02:30 +0000[diff] [blame]94 // Track which registers have been modified and used.
95 BitVector ModifiedRegs, UsedRegs;
96
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]97 // Scan the instructions looking for a load/store that can be combined
98 // with the current instruction into a load/store pair.
99 // Return the matching instruction if one is found, else MBB->end().
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]100 MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
Chad Rosier96a18a92015-07-21 17:42:04 +0000[diff] [blame]101 LdStPairFlags &Flags,
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]102 unsigned Limit,
103 bool FindNarrowMerge);
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]104
105 // Scan the instructions looking for a store that writes to the address from
106 // which the current load instruction reads. Return true if one is found.
107 bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
108 MachineBasicBlock::iterator &StoreI);
109
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]110 // Merge the two instructions indicated into a wider instruction.
111 MachineBasicBlock::iterator
112 mergeNarrowInsns(MachineBasicBlock::iterator I,
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]113 MachineBasicBlock::iterator MergeMI,
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]114 const LdStPairFlags &Flags);
115
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]116 // Merge the two instructions indicated into a single pair-wise instruction.
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]117 MachineBasicBlock::iterator
118 mergePairedInsns(MachineBasicBlock::iterator I,
Chad Rosier96a18a92015-07-21 17:42:04 +0000[diff] [blame]119 MachineBasicBlock::iterator Paired,
Chad Rosierfe5399f2015-07-21 17:47:56 +0000[diff] [blame]120 const LdStPairFlags &Flags);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]121
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]122 // Promote the load that reads directly from the address stored to.
123 MachineBasicBlock::iterator
124 promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
125 MachineBasicBlock::iterator StoreI);
126
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]127 // Scan the instruction list to find a base register update that can
128 // be combined with the current instruction (a load or store) using
129 // pre or post indexed addressing with writeback. Scan forwards.
130 MachineBasicBlock::iterator
Chad Rosier234bf6f2016-01-18 21:56:40 +0000[diff] [blame]131 findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]132 int UnscaledOffset, unsigned Limit);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]133
134 // Scan the instruction list to find a base register update that can
135 // be combined with the current instruction (a load or store) using
136 // pre or post indexed addressing with writeback. Scan backwards.
137 MachineBasicBlock::iterator
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]138 findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]139
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]140 // Find an instruction that updates the base register of the ld/st
141 // instruction.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]142 bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]143 unsigned BaseReg, int Offset);
144
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]145 // Merge a pre- or post-index base register update into a ld/st instruction.
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]146 MachineBasicBlock::iterator
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]147 mergeUpdateInsn(MachineBasicBlock::iterator I,
148 MachineBasicBlock::iterator Update, bool IsPreIdx);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]149
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]150 // Find and merge foldable ldr/str instructions.
151 bool tryToMergeLdStInst(MachineBasicBlock::iterator &MBBI);
152
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]153 // Find and pair ldr/str instructions.
154 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
155
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]156 // Find and promote load instructions which read directly from store.
157 bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
158
Jun Bum Lim22fe15e2015-11-06 16:27:47 +0000[diff] [blame]159 bool optimizeBlock(MachineBasicBlock &MBB, bool enableNarrowLdOpt);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]160
161 bool runOnMachineFunction(MachineFunction &Fn) override;
162
Derek Schuff1dbf7a52016-04-04 17:09:25 +0000[diff] [blame]163 MachineFunctionProperties getRequiredProperties() const override {
164 return MachineFunctionProperties().set(
165 MachineFunctionProperties::Property::AllVRegsAllocated);
166 }
167
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]168 const char *getPassName() const override {
Chad Rosier96530b32015-08-05 13:44:51 +0000[diff] [blame]169 return AARCH64_LOAD_STORE_OPT_NAME;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]170 }
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]171};
172char AArch64LoadStoreOpt::ID = 0;
Jim Grosbach1eee3df2014-08-11 22:42:31 +0000[diff] [blame]173} // namespace
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]174
Chad Rosier96530b32015-08-05 13:44:51 +0000[diff] [blame]175INITIALIZE_PASS(AArch64LoadStoreOpt, "aarch64-ldst-opt",
176 AARCH64_LOAD_STORE_OPT_NAME, false, false)
177
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]178static unsigned getBitExtrOpcode(MachineInstr &MI) {
179 switch (MI.getOpcode()) {
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]180 default:
181 llvm_unreachable("Unexpected opcode.");
182 case AArch64::LDRBBui:
183 case AArch64::LDURBBi:
184 case AArch64::LDRHHui:
185 case AArch64::LDURHHi:
186 return AArch64::UBFMWri;
187 case AArch64::LDRSBWui:
188 case AArch64::LDURSBWi:
189 case AArch64::LDRSHWui:
190 case AArch64::LDURSHWi:
191 return AArch64::SBFMWri;
192 }
193}
194
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]195static bool isNarrowStore(unsigned Opc) {
196 switch (Opc) {
197 default:
198 return false;
199 case AArch64::STRBBui:
200 case AArch64::STURBBi:
201 case AArch64::STRHHui:
202 case AArch64::STURHHi:
203 return true;
204 }
205}
206
Jun Bum Limc12c2792015-11-19 18:41:27 +0000[diff] [blame]207static bool isNarrowLoad(unsigned Opc) {
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]208 switch (Opc) {
209 default:
210 return false;
211 case AArch64::LDRHHui:
212 case AArch64::LDURHHi:
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]213 case AArch64::LDRBBui:
214 case AArch64::LDURBBi:
215 case AArch64::LDRSHWui:
216 case AArch64::LDURSHWi:
217 case AArch64::LDRSBWui:
218 case AArch64::LDURSBWi:
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]219 return true;
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]220 }
221}
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]222
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]223static bool isNarrowLoad(MachineInstr &MI) {
224 return isNarrowLoad(MI.getOpcode());
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]225}
226
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]227static bool isNarrowLoadOrStore(unsigned Opc) {
228 return isNarrowLoad(Opc) || isNarrowStore(Opc);
229}
230
Chad Rosier32d4d372015-09-29 16:07:32 +0000[diff] [blame]231// Scaling factor for unscaled load or store.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]232static int getMemScale(MachineInstr &MI) {
233 switch (MI.getOpcode()) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]234 default:
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]235 llvm_unreachable("Opcode has unknown scale!");
236 case AArch64::LDRBBui:
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]237 case AArch64::LDURBBi:
238 case AArch64::LDRSBWui:
239 case AArch64::LDURSBWi:
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]240 case AArch64::STRBBui:
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]241 case AArch64::STURBBi:
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]242 return 1;
243 case AArch64::LDRHHui:
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]244 case AArch64::LDURHHi:
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]245 case AArch64::LDRSHWui:
246 case AArch64::LDURSHWi:
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]247 case AArch64::STRHHui:
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]248 case AArch64::STURHHi:
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]249 return 2;
Chad Rosiera4d32172015-09-29 14:57:10 +0000[diff] [blame]250 case AArch64::LDRSui:
251 case AArch64::LDURSi:
252 case AArch64::LDRSWui:
253 case AArch64::LDURSWi:
254 case AArch64::LDRWui:
255 case AArch64::LDURWi:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]256 case AArch64::STRSui:
257 case AArch64::STURSi:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]258 case AArch64::STRWui:
259 case AArch64::STURWi:
Chad Rosier32d4d372015-09-29 16:07:32 +0000[diff] [blame]260 case AArch64::LDPSi:
Chad Rosier43150122015-09-29 20:39:55 +0000[diff] [blame]261 case AArch64::LDPSWi:
Chad Rosier32d4d372015-09-29 16:07:32 +0000[diff] [blame]262 case AArch64::LDPWi:
263 case AArch64::STPSi:
264 case AArch64::STPWi:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]265 return 4;
Chad Rosiera4d32172015-09-29 14:57:10 +0000[diff] [blame]266 case AArch64::LDRDui:
267 case AArch64::LDURDi:
268 case AArch64::LDRXui:
269 case AArch64::LDURXi:
270 case AArch64::STRDui:
271 case AArch64::STURDi:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]272 case AArch64::STRXui:
273 case AArch64::STURXi:
Chad Rosier32d4d372015-09-29 16:07:32 +0000[diff] [blame]274 case AArch64::LDPDi:
275 case AArch64::LDPXi:
276 case AArch64::STPDi:
277 case AArch64::STPXi:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]278 return 8;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]279 case AArch64::LDRQui:
280 case AArch64::LDURQi:
Chad Rosiera4d32172015-09-29 14:57:10 +0000[diff] [blame]281 case AArch64::STRQui:
282 case AArch64::STURQi:
Chad Rosier32d4d372015-09-29 16:07:32 +0000[diff] [blame]283 case AArch64::LDPQi:
284 case AArch64::STPQi:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]285 return 16;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]286 }
287}
288
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]289static unsigned getMatchingNonSExtOpcode(unsigned Opc,
290 bool *IsValidLdStrOpc = nullptr) {
291 if (IsValidLdStrOpc)
292 *IsValidLdStrOpc = true;
293 switch (Opc) {
294 default:
295 if (IsValidLdStrOpc)
296 *IsValidLdStrOpc = false;
297 return UINT_MAX;
298 case AArch64::STRDui:
299 case AArch64::STURDi:
300 case AArch64::STRQui:
301 case AArch64::STURQi:
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]302 case AArch64::STRBBui:
303 case AArch64::STURBBi:
304 case AArch64::STRHHui:
305 case AArch64::STURHHi:
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]306 case AArch64::STRWui:
307 case AArch64::STURWi:
308 case AArch64::STRXui:
309 case AArch64::STURXi:
310 case AArch64::LDRDui:
311 case AArch64::LDURDi:
312 case AArch64::LDRQui:
313 case AArch64::LDURQi:
314 case AArch64::LDRWui:
315 case AArch64::LDURWi:
316 case AArch64::LDRXui:
317 case AArch64::LDURXi:
318 case AArch64::STRSui:
319 case AArch64::STURSi:
320 case AArch64::LDRSui:
321 case AArch64::LDURSi:
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]322 case AArch64::LDRHHui:
323 case AArch64::LDURHHi:
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]324 case AArch64::LDRBBui:
325 case AArch64::LDURBBi:
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]326 return Opc;
327 case AArch64::LDRSWui:
328 return AArch64::LDRWui;
329 case AArch64::LDURSWi:
330 return AArch64::LDURWi;
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]331 case AArch64::LDRSBWui:
332 return AArch64::LDRBBui;
333 case AArch64::LDRSHWui:
334 return AArch64::LDRHHui;
335 case AArch64::LDURSBWi:
336 return AArch64::LDURBBi;
337 case AArch64::LDURSHWi:
338 return AArch64::LDURHHi;
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]339 }
340}
341
Jun Bum Lim1de2d442016-02-05 20:02:03 +0000[diff] [blame]342static unsigned getMatchingWideOpcode(unsigned Opc) {
343 switch (Opc) {
344 default:
345 llvm_unreachable("Opcode has no wide equivalent!");
346 case AArch64::STRBBui:
347 return AArch64::STRHHui;
348 case AArch64::STRHHui:
349 return AArch64::STRWui;
350 case AArch64::STURBBi:
351 return AArch64::STURHHi;
352 case AArch64::STURHHi:
353 return AArch64::STURWi;
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]354 case AArch64::STURWi:
355 return AArch64::STURXi;
356 case AArch64::STRWui:
357 return AArch64::STRXui;
Jun Bum Lim1de2d442016-02-05 20:02:03 +0000[diff] [blame]358 case AArch64::LDRHHui:
359 case AArch64::LDRSHWui:
360 return AArch64::LDRWui;
361 case AArch64::LDURHHi:
362 case AArch64::LDURSHWi:
363 return AArch64::LDURWi;
364 case AArch64::LDRBBui:
365 case AArch64::LDRSBWui:
366 return AArch64::LDRHHui;
367 case AArch64::LDURBBi:
368 case AArch64::LDURSBWi:
369 return AArch64::LDURHHi;
370 }
371}
372
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]373static unsigned getMatchingPairOpcode(unsigned Opc) {
374 switch (Opc) {
375 default:
376 llvm_unreachable("Opcode has no pairwise equivalent!");
377 case AArch64::STRSui:
378 case AArch64::STURSi:
379 return AArch64::STPSi;
380 case AArch64::STRDui:
381 case AArch64::STURDi:
382 return AArch64::STPDi;
383 case AArch64::STRQui:
384 case AArch64::STURQi:
385 return AArch64::STPQi;
386 case AArch64::STRWui:
387 case AArch64::STURWi:
388 return AArch64::STPWi;
389 case AArch64::STRXui:
390 case AArch64::STURXi:
391 return AArch64::STPXi;
392 case AArch64::LDRSui:
393 case AArch64::LDURSi:
394 return AArch64::LDPSi;
395 case AArch64::LDRDui:
396 case AArch64::LDURDi:
397 return AArch64::LDPDi;
398 case AArch64::LDRQui:
399 case AArch64::LDURQi:
400 return AArch64::LDPQi;
401 case AArch64::LDRWui:
402 case AArch64::LDURWi:
403 return AArch64::LDPWi;
404 case AArch64::LDRXui:
405 case AArch64::LDURXi:
406 return AArch64::LDPXi;
Quentin Colombet29f55332015-01-24 01:25:54 +0000[diff] [blame]407 case AArch64::LDRSWui:
408 case AArch64::LDURSWi:
409 return AArch64::LDPSWi;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]410 }
411}
412
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]413static unsigned isMatchingStore(MachineInstr &LoadInst,
414 MachineInstr &StoreInst) {
415 unsigned LdOpc = LoadInst.getOpcode();
416 unsigned StOpc = StoreInst.getOpcode();
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]417 switch (LdOpc) {
418 default:
419 llvm_unreachable("Unsupported load instruction!");
420 case AArch64::LDRBBui:
421 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
422 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
423 case AArch64::LDURBBi:
424 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
425 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
426 case AArch64::LDRHHui:
427 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
428 StOpc == AArch64::STRXui;
429 case AArch64::LDURHHi:
430 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
431 StOpc == AArch64::STURXi;
432 case AArch64::LDRWui:
433 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
434 case AArch64::LDURWi:
435 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
436 case AArch64::LDRXui:
437 return StOpc == AArch64::STRXui;
438 case AArch64::LDURXi:
439 return StOpc == AArch64::STURXi;
440 }
441}
442
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]443static unsigned getPreIndexedOpcode(unsigned Opc) {
444 switch (Opc) {
445 default:
446 llvm_unreachable("Opcode has no pre-indexed equivalent!");
Tilmann Scheller5d8d72c2014-06-04 12:40:35 +0000[diff] [blame]447 case AArch64::STRSui:
448 return AArch64::STRSpre;
449 case AArch64::STRDui:
450 return AArch64::STRDpre;
451 case AArch64::STRQui:
452 return AArch64::STRQpre;
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]453 case AArch64::STRBBui:
454 return AArch64::STRBBpre;
455 case AArch64::STRHHui:
456 return AArch64::STRHHpre;
Tilmann Scheller5d8d72c2014-06-04 12:40:35 +0000[diff] [blame]457 case AArch64::STRWui:
458 return AArch64::STRWpre;
459 case AArch64::STRXui:
460 return AArch64::STRXpre;
461 case AArch64::LDRSui:
462 return AArch64::LDRSpre;
463 case AArch64::LDRDui:
464 return AArch64::LDRDpre;
465 case AArch64::LDRQui:
466 return AArch64::LDRQpre;
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]467 case AArch64::LDRBBui:
468 return AArch64::LDRBBpre;
469 case AArch64::LDRHHui:
470 return AArch64::LDRHHpre;
Tilmann Scheller5d8d72c2014-06-04 12:40:35 +0000[diff] [blame]471 case AArch64::LDRWui:
472 return AArch64::LDRWpre;
473 case AArch64::LDRXui:
474 return AArch64::LDRXpre;
Quentin Colombet29f55332015-01-24 01:25:54 +0000[diff] [blame]475 case AArch64::LDRSWui:
476 return AArch64::LDRSWpre;
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]477 case AArch64::LDPSi:
478 return AArch64::LDPSpre;
Chad Rosier43150122015-09-29 20:39:55 +0000[diff] [blame]479 case AArch64::LDPSWi:
480 return AArch64::LDPSWpre;
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]481 case AArch64::LDPDi:
482 return AArch64::LDPDpre;
483 case AArch64::LDPQi:
484 return AArch64::LDPQpre;
485 case AArch64::LDPWi:
486 return AArch64::LDPWpre;
487 case AArch64::LDPXi:
488 return AArch64::LDPXpre;
489 case AArch64::STPSi:
490 return AArch64::STPSpre;
491 case AArch64::STPDi:
492 return AArch64::STPDpre;
493 case AArch64::STPQi:
494 return AArch64::STPQpre;
495 case AArch64::STPWi:
496 return AArch64::STPWpre;
497 case AArch64::STPXi:
498 return AArch64::STPXpre;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]499 }
500}
501
502static unsigned getPostIndexedOpcode(unsigned Opc) {
503 switch (Opc) {
504 default:
505 llvm_unreachable("Opcode has no post-indexed wise equivalent!");
506 case AArch64::STRSui:
507 return AArch64::STRSpost;
508 case AArch64::STRDui:
509 return AArch64::STRDpost;
510 case AArch64::STRQui:
511 return AArch64::STRQpost;
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]512 case AArch64::STRBBui:
513 return AArch64::STRBBpost;
514 case AArch64::STRHHui:
515 return AArch64::STRHHpost;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]516 case AArch64::STRWui:
517 return AArch64::STRWpost;
518 case AArch64::STRXui:
519 return AArch64::STRXpost;
520 case AArch64::LDRSui:
521 return AArch64::LDRSpost;
522 case AArch64::LDRDui:
523 return AArch64::LDRDpost;
524 case AArch64::LDRQui:
525 return AArch64::LDRQpost;
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]526 case AArch64::LDRBBui:
527 return AArch64::LDRBBpost;
528 case AArch64::LDRHHui:
529 return AArch64::LDRHHpost;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]530 case AArch64::LDRWui:
531 return AArch64::LDRWpost;
532 case AArch64::LDRXui:
533 return AArch64::LDRXpost;
Quentin Colombet29f55332015-01-24 01:25:54 +0000[diff] [blame]534 case AArch64::LDRSWui:
535 return AArch64::LDRSWpost;
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]536 case AArch64::LDPSi:
537 return AArch64::LDPSpost;
Chad Rosier43150122015-09-29 20:39:55 +0000[diff] [blame]538 case AArch64::LDPSWi:
539 return AArch64::LDPSWpost;
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]540 case AArch64::LDPDi:
541 return AArch64::LDPDpost;
542 case AArch64::LDPQi:
543 return AArch64::LDPQpost;
544 case AArch64::LDPWi:
545 return AArch64::LDPWpost;
546 case AArch64::LDPXi:
547 return AArch64::LDPXpost;
548 case AArch64::STPSi:
549 return AArch64::STPSpost;
550 case AArch64::STPDi:
551 return AArch64::STPDpost;
552 case AArch64::STPQi:
553 return AArch64::STPQpost;
554 case AArch64::STPWi:
555 return AArch64::STPWpost;
556 case AArch64::STPXi:
557 return AArch64::STPXpost;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]558 }
559}
560
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]561static bool isPairedLdSt(const MachineInstr &MI) {
562 switch (MI.getOpcode()) {
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]563 default:
564 return false;
565 case AArch64::LDPSi:
Chad Rosier43150122015-09-29 20:39:55 +0000[diff] [blame]566 case AArch64::LDPSWi:
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]567 case AArch64::LDPDi:
568 case AArch64::LDPQi:
569 case AArch64::LDPWi:
570 case AArch64::LDPXi:
571 case AArch64::STPSi:
572 case AArch64::STPDi:
573 case AArch64::STPQi:
574 case AArch64::STPWi:
575 case AArch64::STPXi:
576 return true;
577 }
578}
579
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]580static const MachineOperand &getLdStRegOp(const MachineInstr &MI,
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]581 unsigned PairedRegOp = 0) {
582 assert(PairedRegOp < 2 && "Unexpected register operand idx.");
583 unsigned Idx = isPairedLdSt(MI) ? PairedRegOp : 0;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]584 return MI.getOperand(Idx);
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]585}
586
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]587static const MachineOperand &getLdStBaseOp(const MachineInstr &MI) {
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]588 unsigned Idx = isPairedLdSt(MI) ? 2 : 1;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]589 return MI.getOperand(Idx);
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]590}
591
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]592static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI) {
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]593 unsigned Idx = isPairedLdSt(MI) ? 3 : 2;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]594 return MI.getOperand(Idx);
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]595}
596
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]597static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst,
598 MachineInstr &StoreInst,
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]599 const AArch64InstrInfo *TII) {
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]600 assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st.");
601 int LoadSize = getMemScale(LoadInst);
602 int StoreSize = getMemScale(StoreInst);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]603 int UnscaledStOffset = TII->isUnscaledLdSt(StoreInst)
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]604 ? getLdStOffsetOp(StoreInst).getImm()
605 : getLdStOffsetOp(StoreInst).getImm() * StoreSize;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]606 int UnscaledLdOffset = TII->isUnscaledLdSt(LoadInst)
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]607 ? getLdStOffsetOp(LoadInst).getImm()
608 : getLdStOffsetOp(LoadInst).getImm() * LoadSize;
609 return (UnscaledStOffset <= UnscaledLdOffset) &&
610 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
611}
612
Jun Bum Lim33be4992016-05-06 15:08:57 +0000[diff] [blame]613static bool isPromotableZeroStoreOpcode(unsigned Opc) {
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]614 return isNarrowStore(Opc) || Opc == AArch64::STRWui || Opc == AArch64::STURWi;
615}
616
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]617static bool isPromotableZeroStoreOpcode(MachineInstr &MI) {
618 return isPromotableZeroStoreOpcode(MI.getOpcode());
Jun Bum Lim33be4992016-05-06 15:08:57 +0000[diff] [blame]619}
620
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]621static bool isPromotableZeroStoreInst(MachineInstr &MI) {
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]622 return (isPromotableZeroStoreOpcode(MI)) &&
623 getLdStRegOp(MI).getReg() == AArch64::WZR;
624}
625
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]626MachineBasicBlock::iterator
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]627AArch64LoadStoreOpt::mergeNarrowInsns(MachineBasicBlock::iterator I,
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]628 MachineBasicBlock::iterator MergeMI,
Chad Rosier96a18a92015-07-21 17:42:04 +0000[diff] [blame]629 const LdStPairFlags &Flags) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]630 MachineBasicBlock::iterator NextI = I;
631 ++NextI;
632 // If NextI is the second of the two instructions to be merged, we need
633 // to skip one further. Either way we merge will invalidate the iterator,
634 // and we don't need to scan the new instruction, as it's a pairwise
635 // instruction, which we're not considering for further action anyway.
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]636 if (NextI == MergeMI)
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]637 ++NextI;
638
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]639 unsigned Opc = I->getOpcode();
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]640 bool IsScaled = !TII->isUnscaledLdSt(Opc);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]641 int OffsetStride = IsScaled ? 1 : getMemScale(*I);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]642
Chad Rosier96a18a92015-07-21 17:42:04 +0000[diff] [blame]643 bool MergeForward = Flags.getMergeForward();
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]644 // Insert our new paired instruction after whichever of the paired
Tilmann Scheller4aad3bd2014-06-04 12:36:28 +0000[diff] [blame]645 // instructions MergeForward indicates.
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]646 MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
Tilmann Scheller4aad3bd2014-06-04 12:36:28 +0000[diff] [blame]647 // Also based on MergeForward is from where we copy the base register operand
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]648 // so we get the flags compatible with the input code.
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]649 const MachineOperand &BaseRegOp =
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]650 MergeForward ? getLdStBaseOp(*MergeMI) : getLdStBaseOp(*I);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]651
652 // Which register is Rt and which is Rt2 depends on the offset order.
653 MachineInstr *RtMI, *Rt2MI;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]654 if (getLdStOffsetOp(*I).getImm() ==
655 getLdStOffsetOp(*MergeMI).getImm() + OffsetStride) {
656 RtMI = &*MergeMI;
657 Rt2MI = &*I;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]658 } else {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]659 RtMI = &*I;
660 Rt2MI = &*MergeMI;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]661 }
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]662
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]663 int OffsetImm = getLdStOffsetOp(*RtMI).getImm();
Chad Rosier11eedc92016-02-09 19:17:18 +0000[diff] [blame]664 // Change the scaled offset from small to large type.
665 if (IsScaled) {
666 assert(((OffsetImm & 1) == 0) && "Unexpected offset to merge");
667 OffsetImm /= 2;
668 }
669
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]670 DebugLoc DL = I->getDebugLoc();
671 MachineBasicBlock *MBB = I->getParent();
Jun Bum Limc12c2792015-11-19 18:41:27 +0000[diff] [blame]672 if (isNarrowLoad(Opc)) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]673 MachineInstr *RtNewDest = &*(MergeForward ? I : MergeMI);
Oliver Stannardd414c992015-11-10 11:04:18 +0000[diff] [blame]674 // When merging small (< 32 bit) loads for big-endian targets, the order of
675 // the component parts gets swapped.
676 if (!Subtarget->isLittleEndian())
677 std::swap(RtMI, Rt2MI);
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]678 // Construct the new load instruction.
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]679 MachineInstr *NewMemMI, *BitExtMI1, *BitExtMI2;
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]680 NewMemMI =
681 BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingWideOpcode(Opc)))
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]682 .addOperand(getLdStRegOp(*RtNewDest))
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]683 .addOperand(BaseRegOp)
684 .addImm(OffsetImm)
685 .setMemRefs(I->mergeMemRefsWith(*MergeMI));
Chad Rosierf7ac5f22016-03-30 18:08:51 +0000[diff] [blame]686 (void)NewMemMI;
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]687
688 DEBUG(
689 dbgs()
690 << "Creating the new load and extract. Replacing instructions:\n ");
691 DEBUG(I->print(dbgs()));
692 DEBUG(dbgs() << " ");
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]693 DEBUG(MergeMI->print(dbgs()));
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]694 DEBUG(dbgs() << " with instructions:\n ");
695 DEBUG((NewMemMI)->print(dbgs()));
696
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]697 int Width = getMemScale(*I) == 1 ? 8 : 16;
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]698 int LSBLow = 0;
699 int LSBHigh = Width;
700 int ImmsLow = LSBLow + Width - 1;
701 int ImmsHigh = LSBHigh + Width - 1;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]702 MachineInstr *ExtDestMI = &*(MergeForward ? MergeMI : I);
Oliver Stannardd414c992015-11-10 11:04:18 +0000[diff] [blame]703 if ((ExtDestMI == Rt2MI) == Subtarget->isLittleEndian()) {
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]704 // Create the bitfield extract for high bits.
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]705 BitExtMI1 =
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]706 BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*Rt2MI)))
707 .addOperand(getLdStRegOp(*Rt2MI))
708 .addReg(getLdStRegOp(*RtNewDest).getReg())
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]709 .addImm(LSBHigh)
710 .addImm(ImmsHigh);
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]711 // Create the bitfield extract for low bits.
712 if (RtMI->getOpcode() == getMatchingNonSExtOpcode(RtMI->getOpcode())) {
713 // For unsigned, prefer to use AND for low bits.
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]714 BitExtMI2 = BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::ANDWri))
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]715 .addOperand(getLdStRegOp(*RtMI))
716 .addReg(getLdStRegOp(*RtNewDest).getReg())
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]717 .addImm(ImmsLow);
718 } else {
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]719 BitExtMI2 =
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]720 BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*RtMI)))
721 .addOperand(getLdStRegOp(*RtMI))
722 .addReg(getLdStRegOp(*RtNewDest).getReg())
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]723 .addImm(LSBLow)
724 .addImm(ImmsLow);
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]725 }
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]726 } else {
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]727 // Create the bitfield extract for low bits.
728 if (RtMI->getOpcode() == getMatchingNonSExtOpcode(RtMI->getOpcode())) {
729 // For unsigned, prefer to use AND for low bits.
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]730 BitExtMI1 = BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::ANDWri))
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]731 .addOperand(getLdStRegOp(*RtMI))
732 .addReg(getLdStRegOp(*RtNewDest).getReg())
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]733 .addImm(ImmsLow);
734 } else {
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]735 BitExtMI1 =
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]736 BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*RtMI)))
737 .addOperand(getLdStRegOp(*RtMI))
738 .addReg(getLdStRegOp(*RtNewDest).getReg())
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]739 .addImm(LSBLow)
740 .addImm(ImmsLow);
Jun Bum Lim4c35cca2015-11-19 17:21:41 +0000[diff] [blame]741 }
742
743 // Create the bitfield extract for high bits.
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]744 BitExtMI2 =
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]745 BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*Rt2MI)))
746 .addOperand(getLdStRegOp(*Rt2MI))
747 .addReg(getLdStRegOp(*RtNewDest).getReg())
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]748 .addImm(LSBHigh)
749 .addImm(ImmsHigh);
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]750 }
Chad Rosierf7ac5f22016-03-30 18:08:51 +0000[diff] [blame]751 (void)BitExtMI1;
752 (void)BitExtMI2;
753
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]754 DEBUG(dbgs() << " ");
755 DEBUG((BitExtMI1)->print(dbgs()));
756 DEBUG(dbgs() << " ");
757 DEBUG((BitExtMI2)->print(dbgs()));
758 DEBUG(dbgs() << "\n");
759
760 // Erase the old instructions.
761 I->eraseFromParent();
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]762 MergeMI->eraseFromParent();
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]763 return NextI;
764 }
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]765 assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) &&
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]766 "Expected promotable zero store");
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]767
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]768 // Construct the new instruction.
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]769 MachineInstrBuilder MIB;
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]770 MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingWideOpcode(Opc)))
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]771 .addReg(isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]772 .addOperand(BaseRegOp)
773 .addImm(OffsetImm)
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]774 .setMemRefs(I->mergeMemRefsWith(*MergeMI));
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]775 (void)MIB;
776
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]777 DEBUG(dbgs() << "Creating wider load/store. Replacing instructions:\n ");
778 DEBUG(I->print(dbgs()));
779 DEBUG(dbgs() << " ");
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]780 DEBUG(MergeMI->print(dbgs()));
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]781 DEBUG(dbgs() << " with instruction:\n ");
782 DEBUG(((MachineInstr *)MIB)->print(dbgs()));
783 DEBUG(dbgs() << "\n");
784
785 // Erase the old instructions.
786 I->eraseFromParent();
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]787 MergeMI->eraseFromParent();
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]788 return NextI;
789}
790
791MachineBasicBlock::iterator
792AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
793 MachineBasicBlock::iterator Paired,
794 const LdStPairFlags &Flags) {
795 MachineBasicBlock::iterator NextI = I;
796 ++NextI;
797 // If NextI is the second of the two instructions to be merged, we need
798 // to skip one further. Either way we merge will invalidate the iterator,
799 // and we don't need to scan the new instruction, as it's a pairwise
800 // instruction, which we're not considering for further action anyway.
801 if (NextI == Paired)
802 ++NextI;
803
804 int SExtIdx = Flags.getSExtIdx();
805 unsigned Opc =
806 SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(I->getOpcode());
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]807 bool IsUnscaled = TII->isUnscaledLdSt(Opc);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]808 int OffsetStride = IsUnscaled ? getMemScale(*I) : 1;
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]809
810 bool MergeForward = Flags.getMergeForward();
811 // Insert our new paired instruction after whichever of the paired
812 // instructions MergeForward indicates.
813 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
814 // Also based on MergeForward is from where we copy the base register operand
815 // so we get the flags compatible with the input code.
816 const MachineOperand &BaseRegOp =
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]817 MergeForward ? getLdStBaseOp(*Paired) : getLdStBaseOp(*I);
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]818
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]819 int Offset = getLdStOffsetOp(*I).getImm();
820 int PairedOffset = getLdStOffsetOp(*Paired).getImm();
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]821 bool PairedIsUnscaled = TII->isUnscaledLdSt(Paired->getOpcode());
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]822 if (IsUnscaled != PairedIsUnscaled) {
823 // We're trying to pair instructions that differ in how they are scaled. If
824 // I is scaled then scale the offset of Paired accordingly. Otherwise, do
825 // the opposite (i.e., make Paired's offset unscaled).
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]826 int MemSize = getMemScale(*Paired);
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]827 if (PairedIsUnscaled) {
828 // If the unscaled offset isn't a multiple of the MemSize, we can't
829 // pair the operations together.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]830 assert(!(PairedOffset % getMemScale(*Paired)) &&
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]831 "Offset should be a multiple of the stride!");
832 PairedOffset /= MemSize;
833 } else {
834 PairedOffset *= MemSize;
835 }
836 }
837
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]838 // Which register is Rt and which is Rt2 depends on the offset order.
839 MachineInstr *RtMI, *Rt2MI;
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]840 if (Offset == PairedOffset + OffsetStride) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]841 RtMI = &*Paired;
842 Rt2MI = &*I;
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]843 // Here we swapped the assumption made for SExtIdx.
844 // I.e., we turn ldp I, Paired into ldp Paired, I.
845 // Update the index accordingly.
846 if (SExtIdx != -1)
847 SExtIdx = (SExtIdx + 1) % 2;
848 } else {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]849 RtMI = &*I;
850 Rt2MI = &*Paired;
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]851 }
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]852 int OffsetImm = getLdStOffsetOp(*RtMI).getImm();
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]853 // Scale the immediate offset, if necessary.
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]854 if (TII->isUnscaledLdSt(RtMI->getOpcode())) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]855 assert(!(OffsetImm % getMemScale(*RtMI)) &&
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]856 "Unscaled offset cannot be scaled.");
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]857 OffsetImm /= getMemScale(*RtMI);
Chad Rosier87e33412016-02-09 20:18:07 +0000[diff] [blame]858 }
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]859
860 // Construct the new instruction.
861 MachineInstrBuilder MIB;
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]862 DebugLoc DL = I->getDebugLoc();
863 MachineBasicBlock *MBB = I->getParent();
864 MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingPairOpcode(Opc)))
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]865 .addOperand(getLdStRegOp(*RtMI))
866 .addOperand(getLdStRegOp(*Rt2MI))
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]867 .addOperand(BaseRegOp)
Chad Rosiere40b9512016-03-08 17:16:38 +0000[diff] [blame]868 .addImm(OffsetImm)
869 .setMemRefs(I->mergeMemRefsWith(*Paired));
Chad Rosierb5933d72016-02-09 19:02:12 +0000[diff] [blame]870
871 (void)MIB;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]872
873 DEBUG(dbgs() << "Creating pair load/store. Replacing instructions:\n ");
874 DEBUG(I->print(dbgs()));
875 DEBUG(dbgs() << " ");
876 DEBUG(Paired->print(dbgs()));
877 DEBUG(dbgs() << " with instruction:\n ");
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]878 if (SExtIdx != -1) {
879 // Generate the sign extension for the proper result of the ldp.
880 // I.e., with X1, that would be:
881 // %W1<def> = KILL %W1, %X1<imp-def>
882 // %X1<def> = SBFMXri %X1<kill>, 0, 31
883 MachineOperand &DstMO = MIB->getOperand(SExtIdx);
884 // Right now, DstMO has the extended register, since it comes from an
885 // extended opcode.
886 unsigned DstRegX = DstMO.getReg();
887 // Get the W variant of that register.
888 unsigned DstRegW = TRI->getSubReg(DstRegX, AArch64::sub_32);
889 // Update the result of LDP to use the W instead of the X variant.
890 DstMO.setReg(DstRegW);
891 DEBUG(((MachineInstr *)MIB)->print(dbgs()));
892 DEBUG(dbgs() << "\n");
893 // Make the machine verifier happy by providing a definition for
894 // the X register.
895 // Insert this definition right after the generated LDP, i.e., before
896 // InsertionPoint.
897 MachineInstrBuilder MIBKill =
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]898 BuildMI(*MBB, InsertionPoint, DL, TII->get(TargetOpcode::KILL), DstRegW)
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]899 .addReg(DstRegW)
900 .addReg(DstRegX, RegState::Define);
901 MIBKill->getOperand(2).setImplicit();
902 // Create the sign extension.
903 MachineInstrBuilder MIBSXTW =
Chad Rosierc46ef882016-02-09 19:33:42 +0000[diff] [blame]904 BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::SBFMXri), DstRegX)
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]905 .addReg(DstRegX)
906 .addImm(0)
907 .addImm(31);
908 (void)MIBSXTW;
909 DEBUG(dbgs() << " Extend operand:\n ");
910 DEBUG(((MachineInstr *)MIBSXTW)->print(dbgs()));
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]911 } else {
912 DEBUG(((MachineInstr *)MIB)->print(dbgs()));
Quentin Colombet66b61632015-03-06 22:42:10 +0000[diff] [blame]913 }
Chad Rosier1c44c5982016-02-09 20:27:45 +0000[diff] [blame]914 DEBUG(dbgs() << "\n");
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]915
916 // Erase the old instructions.
917 I->eraseFromParent();
918 Paired->eraseFromParent();
919
920 return NextI;
921}
922
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]923MachineBasicBlock::iterator
924AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
925 MachineBasicBlock::iterator StoreI) {
926 MachineBasicBlock::iterator NextI = LoadI;
927 ++NextI;
928
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]929 int LoadSize = getMemScale(*LoadI);
930 int StoreSize = getMemScale(*StoreI);
931 unsigned LdRt = getLdStRegOp(*LoadI).getReg();
932 unsigned StRt = getLdStRegOp(*StoreI).getReg();
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]933 bool IsStoreXReg = TRI->getRegClass(AArch64::GPR64RegClassID)->contains(StRt);
934
935 assert((IsStoreXReg ||
936 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
937 "Unexpected RegClass");
938
939 MachineInstr *BitExtMI;
940 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
941 // Remove the load, if the destination register of the loads is the same
942 // register for stored value.
943 if (StRt == LdRt && LoadSize == 8) {
944 DEBUG(dbgs() << "Remove load instruction:\n ");
945 DEBUG(LoadI->print(dbgs()));
946 DEBUG(dbgs() << "\n");
947 LoadI->eraseFromParent();
948 return NextI;
949 }
950 // Replace the load with a mov if the load and store are in the same size.
951 BitExtMI =
952 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
953 TII->get(IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), LdRt)
954 .addReg(IsStoreXReg ? AArch64::XZR : AArch64::WZR)
955 .addReg(StRt)
956 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
957 } else {
958 // FIXME: Currently we disable this transformation in big-endian targets as
959 // performance and correctness are verified only in little-endian.
960 if (!Subtarget->isLittleEndian())
961 return NextI;
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000[diff] [blame]962 bool IsUnscaled = TII->isUnscaledLdSt(*LoadI);
963 assert(IsUnscaled == TII->isUnscaledLdSt(*StoreI) &&
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]964 "Unsupported ld/st match");
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]965 assert(LoadSize <= StoreSize && "Invalid load size");
966 int UnscaledLdOffset = IsUnscaled
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]967 ? getLdStOffsetOp(*LoadI).getImm()
968 : getLdStOffsetOp(*LoadI).getImm() * LoadSize;
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]969 int UnscaledStOffset = IsUnscaled
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]970 ? getLdStOffsetOp(*StoreI).getImm()
971 : getLdStOffsetOp(*StoreI).getImm() * StoreSize;
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]972 int Width = LoadSize * 8;
973 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
974 int Imms = Immr + Width - 1;
975 unsigned DestReg = IsStoreXReg
976 ? TRI->getMatchingSuperReg(LdRt, AArch64::sub_32,
977 &AArch64::GPR64RegClass)
978 : LdRt;
979
980 assert((UnscaledLdOffset >= UnscaledStOffset &&
981 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
982 "Invalid offset");
983
984 Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
985 Imms = Immr + Width - 1;
986 if (UnscaledLdOffset == UnscaledStOffset) {
987 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N
988 | ((Immr) << 6) // immr
989 | ((Imms) << 0) // imms
990 ;
991
992 BitExtMI =
993 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
994 TII->get(IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
995 DestReg)
996 .addReg(StRt)
997 .addImm(AndMaskEncoded);
998 } else {
999 BitExtMI =
1000 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1001 TII->get(IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1002 DestReg)
1003 .addReg(StRt)
1004 .addImm(Immr)
1005 .addImm(Imms);
1006 }
1007 }
Chad Rosierf7ac5f22016-03-30 18:08:51 +0000[diff] [blame]1008 (void)BitExtMI;
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1009
1010 DEBUG(dbgs() << "Promoting load by replacing :\n ");
1011 DEBUG(StoreI->print(dbgs()));
1012 DEBUG(dbgs() << " ");
1013 DEBUG(LoadI->print(dbgs()));
1014 DEBUG(dbgs() << " with instructions:\n ");
1015 DEBUG(StoreI->print(dbgs()));
1016 DEBUG(dbgs() << " ");
1017 DEBUG((BitExtMI)->print(dbgs()));
1018 DEBUG(dbgs() << "\n");
1019
1020 // Erase the old instructions.
1021 LoadI->eraseFromParent();
1022 return NextI;
1023}
1024
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1025/// trackRegDefsUses - Remember what registers the specified instruction uses
1026/// and modifies.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1027static void trackRegDefsUses(const MachineInstr &MI, BitVector &ModifiedRegs,
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1028 BitVector &UsedRegs,
1029 const TargetRegisterInfo *TRI) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1030 for (const MachineOperand &MO : MI.operands()) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1031 if (MO.isRegMask())
1032 ModifiedRegs.setBitsNotInMask(MO.getRegMask());
1033
1034 if (!MO.isReg())
1035 continue;
1036 unsigned Reg = MO.getReg();
Geoff Berry173b14d2016-02-09 20:47:21 +0000[diff] [blame]1037 if (!Reg)
1038 continue;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1039 if (MO.isDef()) {
1040 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1041 ModifiedRegs.set(*AI);
1042 } else {
1043 assert(MO.isUse() && "Reg operand not a def and not a use?!?");
1044 for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
1045 UsedRegs.set(*AI);
1046 }
1047 }
1048}
1049
1050static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) {
Chad Rosier3dd0e942015-08-18 16:20:03 +0000[diff] [blame]1051 // Convert the byte-offset used by unscaled into an "element" offset used
1052 // by the scaled pair load/store instructions.
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]1053 if (IsUnscaled) {
1054 // If the byte-offset isn't a multiple of the stride, there's no point
1055 // trying to match it.
1056 if (Offset % OffsetStride)
1057 return false;
Chad Rosier3dd0e942015-08-18 16:20:03 +0000[diff] [blame]1058 Offset /= OffsetStride;
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]1059 }
Chad Rosier3dd0e942015-08-18 16:20:03 +0000[diff] [blame]1060 return Offset <= 63 && Offset >= -64;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1061}
1062
1063// Do alignment, specialized to power of 2 and for signed ints,
1064// avoiding having to do a C-style cast from uint_64t to int when
Rui Ueyamada00f2f2016-01-14 21:06:47 +0000[diff] [blame]1065// using alignTo from include/llvm/Support/MathExtras.h.
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1066// FIXME: Move this function to include/MathExtras.h?
1067static int alignTo(int Num, int PowOf2) {
1068 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1069}
1070
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000[diff] [blame]1071static bool mayAlias(MachineInstr &MIa, MachineInstr &MIb,
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1072 const AArch64InstrInfo *TII) {
1073 // One of the instructions must modify memory.
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000[diff] [blame]1074 if (!MIa.mayStore() && !MIb.mayStore())
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1075 return false;
1076
1077 // Both instructions must be memory operations.
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000[diff] [blame]1078 if (!MIa.mayLoadOrStore() && !MIb.mayLoadOrStore())
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1079 return false;
1080
1081 return !TII->areMemAccessesTriviallyDisjoint(MIa, MIb);
1082}
1083
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000[diff] [blame]1084static bool mayAlias(MachineInstr &MIa,
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1085 SmallVectorImpl<MachineInstr *> &MemInsns,
1086 const AArch64InstrInfo *TII) {
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +0000[diff] [blame]1087 for (MachineInstr *MIb : MemInsns)
1088 if (mayAlias(MIa, *MIb, TII))
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1089 return true;
1090
1091 return false;
1092}
1093
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1094bool AArch64LoadStoreOpt::findMatchingStore(
1095 MachineBasicBlock::iterator I, unsigned Limit,
1096 MachineBasicBlock::iterator &StoreI) {
Jun Bum Lim633b2d82016-02-11 16:18:24 +0000[diff] [blame]1097 MachineBasicBlock::iterator B = I->getParent()->begin();
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1098 MachineBasicBlock::iterator MBBI = I;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1099 MachineInstr &LoadMI = *I;
Chad Rosier5c6a66c2016-02-09 15:59:57 +0000[diff] [blame]1100 unsigned BaseReg = getLdStBaseOp(LoadMI).getReg();
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1101
Jun Bum Lim633b2d82016-02-11 16:18:24 +0000[diff] [blame]1102 // If the load is the first instruction in the block, there's obviously
1103 // not any matching store.
1104 if (MBBI == B)
1105 return false;
1106
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1107 // Track which registers have been modified and used between the first insn
1108 // and the second insn.
Chad Rosierbba881e2016-02-02 15:02:30 +0000[diff] [blame]1109 ModifiedRegs.reset();
1110 UsedRegs.reset();
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1111
Jun Bum Lim633b2d82016-02-11 16:18:24 +0000[diff] [blame]1112 unsigned Count = 0;
1113 do {
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1114 --MBBI;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1115 MachineInstr &MI = *MBBI;
Jun Bum Lim633b2d82016-02-11 16:18:24 +0000[diff] [blame]1116
Geoff Berry4ff2e362016-07-21 15:20:25 +0000[diff] [blame]1117 // Don't count transient instructions towards the search limit since there
1118 // may be different numbers of them if e.g. debug information is present.
1119 if (!MI.isTransient())
Jun Bum Lim633b2d82016-02-11 16:18:24 +0000[diff] [blame]1120 ++Count;
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1121
1122 // If the load instruction reads directly from the address to which the
1123 // store instruction writes and the stored value is not modified, we can
1124 // promote the load. Since we do not handle stores with pre-/post-index,
1125 // it's unnecessary to check if BaseReg is modified by the store itself.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1126 if (MI.mayStore() && isMatchingStore(LoadMI, MI) &&
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1127 BaseReg == getLdStBaseOp(MI).getReg() &&
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]1128 isLdOffsetInRangeOfSt(LoadMI, MI, TII) &&
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1129 !ModifiedRegs[getLdStRegOp(MI).getReg()]) {
1130 StoreI = MBBI;
1131 return true;
1132 }
1133
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1134 if (MI.isCall())
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1135 return false;
1136
1137 // Update modified / uses register lists.
1138 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
1139
1140 // Otherwise, if the base register is modified, we have no match, so
1141 // return early.
1142 if (ModifiedRegs[BaseReg])
1143 return false;
1144
1145 // If we encounter a store aliased with the load, return early.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1146 if (MI.mayStore() && mayAlias(LoadMI, MI, TII))
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1147 return false;
Jun Bum Lim633b2d82016-02-11 16:18:24 +0000[diff] [blame]1148 } while (MBBI != B && Count < Limit);
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1149 return false;
1150}
1151
Chad Rosierc5083c22016-06-10 20:47:14 +0000[diff] [blame]1152// Returns true if FirstMI and MI are candidates for merging or pairing.
1153// Otherwise, returns false.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1154static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
Chad Rosierc5083c22016-06-10 20:47:14 +0000[diff] [blame]1155 LdStPairFlags &Flags,
1156 const AArch64InstrInfo *TII) {
1157 // If this is volatile or if pairing is suppressed, not a candidate.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1158 if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI))
Chad Rosierc5083c22016-06-10 20:47:14 +0000[diff] [blame]1159 return false;
1160
1161 // We should have already checked FirstMI for pair suppression and volatility.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1162 assert(!FirstMI.hasOrderedMemoryRef() &&
1163 !TII->isLdStPairSuppressed(FirstMI) &&
Chad Rosierc5083c22016-06-10 20:47:14 +0000[diff] [blame]1164 "FirstMI shouldn't get here if either of these checks are true.");
1165
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1166 unsigned OpcA = FirstMI.getOpcode();
1167 unsigned OpcB = MI.getOpcode();
Chad Rosierc5083c22016-06-10 20:47:14 +0000[diff] [blame]1168
Chad Rosierc3f6cb92016-02-10 19:45:48 +0000[diff] [blame]1169 // Opcodes match: nothing more to check.
1170 if (OpcA == OpcB)
1171 return true;
1172
1173 // Try to match a sign-extended load/store with a zero-extended load/store.
1174 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1175 unsigned NonSExtOpc = getMatchingNonSExtOpcode(OpcA, &IsValidLdStrOpc);
1176 assert(IsValidLdStrOpc &&
1177 "Given Opc should be a Load or Store with an immediate");
1178 // OpcA will be the first instruction in the pair.
1179 if (NonSExtOpc == getMatchingNonSExtOpcode(OpcB, &PairIsValidLdStrOpc)) {
1180 Flags.setSExtIdx(NonSExtOpc == (unsigned)OpcA ? 1 : 0);
1181 return true;
1182 }
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]1183
1184 // If the second instruction isn't even a load/store, bail out.
1185 if (!PairIsValidLdStrOpc)
1186 return false;
1187
1188 // FIXME: We don't support merging narrow loads/stores with mixed
1189 // scaled/unscaled offsets.
1190 if (isNarrowLoadOrStore(OpcA) || isNarrowLoadOrStore(OpcB))
1191 return false;
1192
1193 // Try to match an unscaled load/store with a scaled load/store.
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]1194 return TII->isUnscaledLdSt(OpcA) != TII->isUnscaledLdSt(OpcB) &&
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]1195 getMatchingPairOpcode(OpcA) == getMatchingPairOpcode(OpcB);
1196
1197 // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
Chad Rosierc3f6cb92016-02-10 19:45:48 +0000[diff] [blame]1198}
1199
Chad Rosier9f4ec2e2016-02-10 18:49:28 +0000[diff] [blame]1200/// Scan the instructions looking for a load/store that can be combined with the
1201/// current instruction into a wider equivalent or a load/store pair.
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1202MachineBasicBlock::iterator
1203AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1204 LdStPairFlags &Flags, unsigned Limit,
1205 bool FindNarrowMerge) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1206 MachineBasicBlock::iterator E = I->getParent()->end();
1207 MachineBasicBlock::iterator MBBI = I;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1208 MachineInstr &FirstMI = *I;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1209 ++MBBI;
1210
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1211 bool MayLoad = FirstMI.mayLoad();
1212 bool IsUnscaled = TII->isUnscaledLdSt(FirstMI);
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]1213 unsigned Reg = getLdStRegOp(FirstMI).getReg();
1214 unsigned BaseReg = getLdStBaseOp(FirstMI).getReg();
1215 int Offset = getLdStOffsetOp(FirstMI).getImm();
Chad Rosierf11d0402015-10-01 18:17:12 +0000[diff] [blame]1216 int OffsetStride = IsUnscaled ? getMemScale(FirstMI) : 1;
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]1217 bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1218
1219 // Track which registers have been modified and used between the first insn
1220 // (inclusive) and the second insn.
Chad Rosierbba881e2016-02-02 15:02:30 +0000[diff] [blame]1221 ModifiedRegs.reset();
1222 UsedRegs.reset();
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1223
1224 // Remember any instructions that read/write memory between FirstMI and MI.
1225 SmallVector<MachineInstr *, 4> MemInsns;
1226
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1227 for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1228 MachineInstr &MI = *MBBI;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1229
Geoff Berry4ff2e362016-07-21 15:20:25 +0000[diff] [blame]1230 // Don't count transient instructions towards the search limit since there
1231 // may be different numbers of them if e.g. debug information is present.
1232 if (!MI.isTransient())
1233 ++Count;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1234
Chad Rosier18896c02016-02-04 16:01:40 +0000[diff] [blame]1235 Flags.setSExtIdx(-1);
Chad Rosierc5083c22016-06-10 20:47:14 +0000[diff] [blame]1236 if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
Chad Rosierc3f6cb92016-02-10 19:45:48 +0000[diff] [blame]1237 getLdStOffsetOp(MI).isImm()) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1238 assert(MI.mayLoadOrStore() && "Expected memory operation.");
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1239 // If we've found another instruction with the same opcode, check to see
1240 // if the base and offset are compatible with our starting instruction.
1241 // These instructions all have scaled immediate operands, so we just
1242 // check for +1/-1. Make sure to check the new instruction offset is
1243 // actually an immediate and not a symbolic reference destined for
1244 // a relocation.
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]1245 unsigned MIBaseReg = getLdStBaseOp(MI).getReg();
1246 int MIOffset = getLdStOffsetOp(MI).getImm();
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1247 bool MIIsUnscaled = TII->isUnscaledLdSt(MI);
Chad Rosier00f9d232016-02-11 14:25:08 +0000[diff] [blame]1248 if (IsUnscaled != MIIsUnscaled) {
1249 // We're trying to pair instructions that differ in how they are scaled.
1250 // If FirstMI is scaled then scale the offset of MI accordingly.
1251 // Otherwise, do the opposite (i.e., make MI's offset unscaled).
1252 int MemSize = getMemScale(MI);
1253 if (MIIsUnscaled) {
1254 // If the unscaled offset isn't a multiple of the MemSize, we can't
1255 // pair the operations together: bail and keep looking.
1256 if (MIOffset % MemSize)
1257 continue;
1258 MIOffset /= MemSize;
1259 } else {
1260 MIOffset *= MemSize;
1261 }
1262 }
1263
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1264 if (BaseReg == MIBaseReg && ((Offset == MIOffset + OffsetStride) ||
1265 (Offset + OffsetStride == MIOffset))) {
1266 int MinOffset = Offset < MIOffset ? Offset : MIOffset;
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1267 if (FindNarrowMerge) {
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]1268 // If the alignment requirements of the scaled wide load/store
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1269 // instruction can't express the offset of the scaled narrow input,
1270 // bail and keep looking. For promotable zero stores, allow only when
1271 // the stored value is the same (i.e., WZR).
1272 if ((!IsUnscaled && alignTo(MinOffset, 2) != MinOffset) ||
1273 (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) {
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1274 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1275 MemInsns.push_back(&MI);
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1276 continue;
1277 }
1278 } else {
Chad Rosierd1f6c842016-06-10 20:49:18 +0000[diff] [blame]1279 // Pairwise instructions have a 7-bit signed offset field. Single
1280 // insns have a 12-bit unsigned offset field. If the resultant
1281 // immediate offset of merging these instructions is out of range for
1282 // a pairwise instruction, bail and keep looking.
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1283 if (!inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) {
1284 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1285 MemInsns.push_back(&MI);
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1286 continue;
1287 }
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1288 // If the alignment requirements of the paired (scaled) instruction
1289 // can't express the offset of the unscaled input, bail and keep
1290 // looking.
1291 if (IsUnscaled && (alignTo(MinOffset, OffsetStride) != MinOffset)) {
1292 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1293 MemInsns.push_back(&MI);
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1294 continue;
1295 }
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1296 }
1297 // If the destination register of the loads is the same register, bail
1298 // and keep looking. A load-pair instruction with both destination
1299 // registers the same is UNPREDICTABLE and will result in an exception.
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1300 if (MayLoad && Reg == getLdStRegOp(MI).getReg()) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1301 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1302 MemInsns.push_back(&MI);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1303 continue;
1304 }
1305
1306 // If the Rt of the second instruction was not modified or used between
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1307 // the two instructions and none of the instructions between the second
1308 // and first alias with the second, we can combine the second into the
1309 // first.
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]1310 if (!ModifiedRegs[getLdStRegOp(MI).getReg()] &&
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1311 !(MI.mayLoad() && UsedRegs[getLdStRegOp(MI).getReg()]) &&
1312 !mayAlias(MI, MemInsns, TII)) {
Chad Rosier96a18a92015-07-21 17:42:04 +0000[diff] [blame]1313 Flags.setMergeForward(false);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1314 return MBBI;
1315 }
1316
1317 // Likewise, if the Rt of the first instruction is not modified or used
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1318 // between the two instructions and none of the instructions between the
1319 // first and the second alias with the first, we can combine the first
1320 // into the second.
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]1321 if (!ModifiedRegs[getLdStRegOp(FirstMI).getReg()] &&
Chad Rosier5f668e12015-09-03 14:19:43 +0000[diff] [blame]1322 !(MayLoad && UsedRegs[getLdStRegOp(FirstMI).getReg()]) &&
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1323 !mayAlias(FirstMI, MemInsns, TII)) {
Chad Rosier96a18a92015-07-21 17:42:04 +0000[diff] [blame]1324 Flags.setMergeForward(true);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1325 return MBBI;
1326 }
1327 // Unable to combine these instructions due to interference in between.
1328 // Keep looking.
1329 }
1330 }
1331
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1332 // If the instruction wasn't a matching load or store. Stop searching if we
1333 // encounter a call instruction that might modify memory.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1334 if (MI.isCall())
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1335 return E;
1336
1337 // Update modified / uses register lists.
1338 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
1339
1340 // Otherwise, if the base register is modified, we have no match, so
1341 // return early.
1342 if (ModifiedRegs[BaseReg])
1343 return E;
Chad Rosierce8e5ab2015-05-21 21:36:46 +0000[diff] [blame]1344
1345 // Update list of instructions that read/write memory.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1346 if (MI.mayLoadOrStore())
1347 MemInsns.push_back(&MI);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1348 }
1349 return E;
1350}
1351
1352MachineBasicBlock::iterator
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]1353AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I,
1354 MachineBasicBlock::iterator Update,
1355 bool IsPreIdx) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1356 assert((Update->getOpcode() == AArch64::ADDXri ||
1357 Update->getOpcode() == AArch64::SUBXri) &&
1358 "Unexpected base register update instruction to merge!");
1359 MachineBasicBlock::iterator NextI = I;
1360 // Return the instruction following the merged instruction, which is
1361 // the instruction following our unmerged load. Unless that's the add/sub
1362 // instruction we're merging, in which case it's the one after that.
1363 if (++NextI == Update)
1364 ++NextI;
1365
1366 int Value = Update->getOperand(2).getImm();
1367 assert(AArch64_AM::getShiftValue(Update->getOperand(3).getImm()) == 0 &&
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]1368 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1369 if (Update->getOpcode() == AArch64::SUBXri)
1370 Value = -Value;
1371
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]1372 unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(I->getOpcode())
1373 : getPostIndexedOpcode(I->getOpcode());
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1374 MachineInstrBuilder MIB;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1375 if (!isPairedLdSt(*I)) {
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1376 // Non-paired instruction.
1377 MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1378 .addOperand(getLdStRegOp(*Update))
1379 .addOperand(getLdStRegOp(*I))
1380 .addOperand(getLdStBaseOp(*I))
Chad Rosier3ada75f2016-01-28 15:38:24 +0000[diff] [blame]1381 .addImm(Value)
1382 .setMemRefs(I->memoperands_begin(), I->memoperands_end());
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1383 } else {
1384 // Paired instruction.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1385 int Scale = getMemScale(*I);
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1386 MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1387 .addOperand(getLdStRegOp(*Update))
1388 .addOperand(getLdStRegOp(*I, 0))
1389 .addOperand(getLdStRegOp(*I, 1))
1390 .addOperand(getLdStBaseOp(*I))
Chad Rosier3ada75f2016-01-28 15:38:24 +0000[diff] [blame]1391 .addImm(Value / Scale)
1392 .setMemRefs(I->memoperands_begin(), I->memoperands_end());
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1393 }
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1394 (void)MIB;
1395
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]1396 if (IsPreIdx)
1397 DEBUG(dbgs() << "Creating pre-indexed load/store.");
1398 else
1399 DEBUG(dbgs() << "Creating post-indexed load/store.");
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1400 DEBUG(dbgs() << " Replacing instructions:\n ");
1401 DEBUG(I->print(dbgs()));
1402 DEBUG(dbgs() << " ");
1403 DEBUG(Update->print(dbgs()));
1404 DEBUG(dbgs() << " with instruction:\n ");
1405 DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1406 DEBUG(dbgs() << "\n");
1407
1408 // Erase the old instructions for the block.
1409 I->eraseFromParent();
1410 Update->eraseFromParent();
1411
1412 return NextI;
1413}
1414
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1415bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
1416 MachineInstr &MI,
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1417 unsigned BaseReg, int Offset) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1418 switch (MI.getOpcode()) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1419 default:
1420 break;
1421 case AArch64::SUBXri:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1422 case AArch64::ADDXri:
1423 // Make sure it's a vanilla immediate operand, not a relocation or
1424 // anything else we can't handle.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1425 if (!MI.getOperand(2).isImm())
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1426 break;
1427 // Watch out for 1 << 12 shifted value.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1428 if (AArch64_AM::getShiftValue(MI.getOperand(3).getImm()))
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1429 break;
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1430
1431 // The update instruction source and destination register must be the
1432 // same as the load/store base register.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1433 if (MI.getOperand(0).getReg() != BaseReg ||
1434 MI.getOperand(1).getReg() != BaseReg)
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1435 break;
1436
1437 bool IsPairedInsn = isPairedLdSt(MemMI);
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1438 int UpdateOffset = MI.getOperand(2).getImm();
Eli Friedman8585e9d2016-08-12 20:28:02 +0000[diff] [blame]1439 if (MI.getOpcode() == AArch64::SUBXri)
1440 UpdateOffset = -UpdateOffset;
1441
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1442 // For non-paired load/store instructions, the immediate must fit in a
1443 // signed 9-bit integer.
1444 if (!IsPairedInsn && (UpdateOffset > 255 || UpdateOffset < -256))
1445 break;
1446
1447 // For paired load/store instructions, the immediate must be a multiple of
1448 // the scaling factor. The scaled offset must also fit into a signed 7-bit
1449 // integer.
1450 if (IsPairedInsn) {
Chad Rosier32d4d372015-09-29 16:07:32 +0000[diff] [blame]1451 int Scale = getMemScale(MemMI);
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1452 if (UpdateOffset % Scale != 0)
1453 break;
1454
1455 int ScaledOffset = UpdateOffset / Scale;
Eli Friedman8585e9d2016-08-12 20:28:02 +0000[diff] [blame]1456 if (ScaledOffset > 63 || ScaledOffset < -64)
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1457 break;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1458 }
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1459
1460 // If we have a non-zero Offset, we check that it matches the amount
1461 // we're adding to the register.
Eli Friedman8585e9d2016-08-12 20:28:02 +0000[diff] [blame]1462 if (!Offset || Offset == UpdateOffset)
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1463 return true;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1464 break;
1465 }
1466 return false;
1467}
1468
1469MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1470 MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1471 MachineBasicBlock::iterator E = I->getParent()->end();
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1472 MachineInstr &MemMI = *I;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1473 MachineBasicBlock::iterator MBBI = I;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1474
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]1475 unsigned BaseReg = getLdStBaseOp(MemMI).getReg();
Chad Rosier0b15e7c2015-10-01 13:33:31 +0000[diff] [blame]1476 int MIUnscaledOffset = getLdStOffsetOp(MemMI).getImm() * getMemScale(MemMI);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1477
Chad Rosierb7c5b912015-10-01 13:43:05 +0000[diff] [blame]1478 // Scan forward looking for post-index opportunities. Updating instructions
1479 // can't be formed if the memory instruction doesn't have the offset we're
1480 // looking for.
1481 if (MIUnscaledOffset != UnscaledOffset)
1482 return E;
1483
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1484 // If the base register overlaps a destination register, we can't
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1485 // merge the update.
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1486 bool IsPairedInsn = isPairedLdSt(MemMI);
1487 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
1488 unsigned DestReg = getLdStRegOp(MemMI, i).getReg();
1489 if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
1490 return E;
1491 }
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1492
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1493 // Track which registers have been modified and used between the first insn
1494 // (inclusive) and the second insn.
Chad Rosierbba881e2016-02-02 15:02:30 +0000[diff] [blame]1495 ModifiedRegs.reset();
1496 UsedRegs.reset();
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1497 ++MBBI;
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1498 for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1499 MachineInstr &MI = *MBBI;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1500
Geoff Berry4ff2e362016-07-21 15:20:25 +0000[diff] [blame]1501 // Don't count transient instructions towards the search limit since there
1502 // may be different numbers of them if e.g. debug information is present.
1503 if (!MI.isTransient())
1504 ++Count;
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1505
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1506 // If we found a match, return it.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1507 if (isMatchingUpdateInsn(*I, MI, BaseReg, UnscaledOffset))
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1508 return MBBI;
1509
1510 // Update the status of what the instruction clobbered and used.
1511 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
1512
1513 // Otherwise, if the base register is used or modified, we have no match, so
1514 // return early.
1515 if (ModifiedRegs[BaseReg] || UsedRegs[BaseReg])
1516 return E;
1517 }
1518 return E;
1519}
1520
1521MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1522 MachineBasicBlock::iterator I, unsigned Limit) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1523 MachineBasicBlock::iterator B = I->getParent()->begin();
1524 MachineBasicBlock::iterator E = I->getParent()->end();
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1525 MachineInstr &MemMI = *I;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1526 MachineBasicBlock::iterator MBBI = I;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1527
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]1528 unsigned BaseReg = getLdStBaseOp(MemMI).getReg();
1529 int Offset = getLdStOffsetOp(MemMI).getImm();
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1530
1531 // If the load/store is the first instruction in the block, there's obviously
1532 // not any matching update. Ditto if the memory offset isn't zero.
1533 if (MBBI == B || Offset != 0)
1534 return E;
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1535 // If the base register overlaps a destination register, we can't
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1536 // merge the update.
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1537 bool IsPairedInsn = isPairedLdSt(MemMI);
1538 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
1539 unsigned DestReg = getLdStRegOp(MemMI, i).getReg();
1540 if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
1541 return E;
1542 }
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1543
1544 // Track which registers have been modified and used between the first insn
1545 // (inclusive) and the second insn.
Chad Rosierbba881e2016-02-02 15:02:30 +0000[diff] [blame]1546 ModifiedRegs.reset();
1547 UsedRegs.reset();
Geoff Berry173b14d2016-02-09 20:47:21 +0000[diff] [blame]1548 unsigned Count = 0;
1549 do {
1550 --MBBI;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1551 MachineInstr &MI = *MBBI;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1552
Geoff Berry4ff2e362016-07-21 15:20:25 +0000[diff] [blame]1553 // Don't count transient instructions towards the search limit since there
1554 // may be different numbers of them if e.g. debug information is present.
1555 if (!MI.isTransient())
Geoff Berry173b14d2016-02-09 20:47:21 +0000[diff] [blame]1556 ++Count;
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1557
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1558 // If we found a match, return it.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1559 if (isMatchingUpdateInsn(*I, MI, BaseReg, Offset))
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1560 return MBBI;
1561
1562 // Update the status of what the instruction clobbered and used.
1563 trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
1564
1565 // Otherwise, if the base register is used or modified, we have no match, so
1566 // return early.
1567 if (ModifiedRegs[BaseReg] || UsedRegs[BaseReg])
1568 return E;
Geoff Berry173b14d2016-02-09 20:47:21 +0000[diff] [blame]1569 } while (MBBI != B && Count < Limit);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1570 return E;
1571}
1572
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1573bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
1574 MachineBasicBlock::iterator &MBBI) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1575 MachineInstr &MI = *MBBI;
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1576 // If this is a volatile load, don't mess with it.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1577 if (MI.hasOrderedMemoryRef())
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1578 return false;
1579
1580 // Make sure this is a reg+imm.
1581 // FIXME: It is possible to extend it to handle reg+reg cases.
1582 if (!getLdStOffsetOp(MI).isImm())
1583 return false;
1584
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1585 // Look backward up to LdStLimit instructions.
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1586 MachineBasicBlock::iterator StoreI;
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1587 if (findMatchingStore(MBBI, LdStLimit, StoreI)) {
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1588 ++NumLoadsFromStoresPromoted;
1589 // Promote the load. Keeping the iterator straight is a
1590 // pain, so we let the merge routine tell us what the next instruction
1591 // is after it's done mucking about.
1592 MBBI = promoteLoadFromStore(MBBI, StoreI);
1593 return true;
1594 }
1595 return false;
1596}
1597
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1598// Find narrow loads that can be converted into a single wider load with
1599// bitfield extract instructions. Also merge adjacent zero stores into a wider
1600// store.
1601bool AArch64LoadStoreOpt::tryToMergeLdStInst(
1602 MachineBasicBlock::iterator &MBBI) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1603 assert((isNarrowLoad(*MBBI) || isPromotableZeroStoreOpcode(*MBBI)) &&
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]1604 "Expected narrow op.");
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1605 MachineInstr &MI = *MBBI;
1606 MachineBasicBlock::iterator E = MI.getParent()->end();
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1607
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1608 if (!TII->isCandidateToMergeOrPair(MI))
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1609 return false;
1610
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]1611 // For promotable zero stores, the stored value should be WZR.
1612 if (isPromotableZeroStoreOpcode(MI) &&
1613 getLdStRegOp(MI).getReg() != AArch64::WZR)
Chad Rosierf7cd8ea2016-02-09 21:20:12 +0000[diff] [blame]1614 return false;
1615
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1616 // Look ahead up to LdStLimit instructions for a mergable instruction.
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1617 LdStPairFlags Flags;
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]1618 MachineBasicBlock::iterator MergeMI =
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1619 findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ true);
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]1620 if (MergeMI != E) {
Jun Bum Limc12c2792015-11-19 18:41:27 +0000[diff] [blame]1621 if (isNarrowLoad(MI)) {
1622 ++NumNarrowLoadsPromoted;
Jun Bum Lim397eb7b2016-02-12 15:25:39 +0000[diff] [blame]1623 } else if (isPromotableZeroStoreInst(MI)) {
Jun Bum Lim80ec0d32015-11-20 21:14:07 +0000[diff] [blame]1624 ++NumZeroStoresPromoted;
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1625 }
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1626 // Keeping the iterator straight is a pain, so we let the merge routine tell
1627 // us what the next instruction is after it's done mucking about.
Chad Rosierd7363db2016-02-09 19:09:22 +0000[diff] [blame]1628 MBBI = mergeNarrowInsns(MBBI, MergeMI, Flags);
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1629 return true;
1630 }
1631 return false;
1632}
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1633
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1634// Find loads and stores that can be merged into a single load or store pair
1635// instruction.
1636bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1637 MachineInstr &MI = *MBBI;
1638 MachineBasicBlock::iterator E = MI.getParent()->end();
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1639
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1640 if (!TII->isCandidateToMergeOrPair(MI))
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1641 return false;
1642
Chad Rosierfc3bf1f2016-02-10 15:52:46 +0000[diff] [blame]1643 // Early exit if the offset is not possible to match. (6 bits of positive
1644 // range, plus allow an extra one in case we find a later insn that matches
1645 // with Offset-1)
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1646 bool IsUnscaled = TII->isUnscaledLdSt(MI);
Chad Rosierfc3bf1f2016-02-10 15:52:46 +0000[diff] [blame]1647 int Offset = getLdStOffsetOp(MI).getImm();
1648 int OffsetStride = IsUnscaled ? getMemScale(MI) : 1;
1649 if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
1650 return false;
1651
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1652 // Look ahead up to LdStLimit instructions for a pairable instruction.
1653 LdStPairFlags Flags;
Jun Bum Limcf974432016-03-31 14:47:24 +0000[diff] [blame]1654 MachineBasicBlock::iterator Paired =
1655 findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ false);
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1656 if (Paired != E) {
1657 ++NumPairCreated;
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1658 if (TII->isUnscaledLdSt(MI))
Chad Rosier24c46ad2016-02-09 18:10:20 +0000[diff] [blame]1659 ++NumUnscaledPairCreated;
1660 // Keeping the iterator straight is a pain, so we let the merge routine tell
1661 // us what the next instruction is after it's done mucking about.
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1662 MBBI = mergePairedInsns(MBBI, Paired, Flags);
1663 return true;
1664 }
1665 return false;
1666}
1667
Jun Bum Lim22fe15e2015-11-06 16:27:47 +0000[diff] [blame]1668bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
1669 bool enableNarrowLdOpt) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1670 bool Modified = false;
Chad Rosierdbdb1d62016-02-01 21:38:31 +0000[diff] [blame]1671 // Four tranformations to do here:
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1672 // 1) Find loads that directly read from stores and promote them by
1673 // replacing with mov instructions. If the store is wider than the load,
1674 // the load will be replaced with a bitfield extract.
1675 // e.g.,
1676 // str w1, [x0, #4]
1677 // ldrh w2, [x0, #6]
1678 // ; becomes
1679 // str w1, [x0, #4]
NAKAMURA Takumife1202c2016-06-20 00:37:41 +0000[diff] [blame]1680 // lsr w2, w1, #16
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1681 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1682 MBBI != E;) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1683 MachineInstr &MI = *MBBI;
1684 switch (MI.getOpcode()) {
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1685 default:
1686 // Just move on to the next instruction.
1687 ++MBBI;
1688 break;
1689 // Scaled instructions.
1690 case AArch64::LDRBBui:
1691 case AArch64::LDRHHui:
1692 case AArch64::LDRWui:
1693 case AArch64::LDRXui:
1694 // Unscaled instructions.
1695 case AArch64::LDURBBi:
1696 case AArch64::LDURHHi:
1697 case AArch64::LDURWi:
1698 case AArch64::LDURXi: {
1699 if (tryToPromoteLoadFromStore(MBBI)) {
1700 Modified = true;
1701 break;
1702 }
1703 ++MBBI;
1704 break;
1705 }
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1706 }
1707 }
Chad Rosierdbdb1d62016-02-01 21:38:31 +0000[diff] [blame]1708 // 2) Find narrow loads that can be converted into a single wider load
1709 // with bitfield extract instructions.
1710 // e.g.,
1711 // ldrh w0, [x2]
1712 // ldrh w1, [x2, #2]
1713 // ; becomes
1714 // ldr w0, [x2]
1715 // ubfx w1, w0, #16, #16
1716 // and w0, w0, #ffff
Jun Bum Lim1de2d442016-02-05 20:02:03 +0000[diff] [blame]1717 //
1718 // Also merge adjacent zero stores into a wider store.
1719 // e.g.,
1720 // strh wzr, [x0]
1721 // strh wzr, [x0, #2]
1722 // ; becomes
1723 // str wzr, [x0]
Jun Bum Lim6755c3b2015-12-22 16:36:16 +0000[diff] [blame]1724 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
Jun Bum Lim22fe15e2015-11-06 16:27:47 +0000[diff] [blame]1725 enableNarrowLdOpt && MBBI != E;) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1726 MachineInstr &MI = *MBBI;
1727 unsigned Opc = MI.getOpcode();
Jun Bum Lim33be4992016-05-06 15:08:57 +0000[diff] [blame]1728 if (isPromotableZeroStoreOpcode(Opc) ||
1729 (EnableNarrowLdMerge && isNarrowLoad(Opc))) {
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1730 if (tryToMergeLdStInst(MBBI)) {
1731 Modified = true;
Jun Bum Lim33be4992016-05-06 15:08:57 +0000[diff] [blame]1732 } else
1733 ++MBBI;
1734 } else
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1735 ++MBBI;
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1736 }
Jun Bum Lim33be4992016-05-06 15:08:57 +0000[diff] [blame]1737
Chad Rosierdbdb1d62016-02-01 21:38:31 +0000[diff] [blame]1738 // 3) Find loads and stores that can be merged into a single load or store
1739 // pair instruction.
1740 // e.g.,
1741 // ldr x0, [x2]
1742 // ldr x1, [x2, #8]
1743 // ; becomes
1744 // ldp x0, x1, [x2]
Jun Bum Limc9879ec2015-10-27 19:16:03 +0000[diff] [blame]1745 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1746 MBBI != E;) {
Geoff Berry22dfbc52016-08-12 15:26:00 +0000[diff] [blame]1747 if (TII->isPairableLdStInst(*MBBI) && tryToPairLdStInst(MBBI))
1748 Modified = true;
1749 else
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1750 ++MBBI;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1751 }
Chad Rosierdbdb1d62016-02-01 21:38:31 +0000[diff] [blame]1752 // 4) Find base register updates that can be merged into the load or store
1753 // as a base-reg writeback.
1754 // e.g.,
1755 // ldr x0, [x2]
1756 // add x2, x2, #4
1757 // ; becomes
1758 // ldr x0, [x2], #4
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1759 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
1760 MBBI != E;) {
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1761 MachineInstr &MI = *MBBI;
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1762 // Do update merging. It's simpler to keep this separate from the above
Chad Rosierdbdb1d62016-02-01 21:38:31 +0000[diff] [blame]1763 // switchs, though not strictly necessary.
Duncan P. N. Exon Smithab53fd92016-07-08 20:29:42 +0000[diff] [blame]1764 unsigned Opc = MI.getOpcode();
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1765 switch (Opc) {
1766 default:
1767 // Just move on to the next instruction.
1768 ++MBBI;
1769 break;
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1770 // Scaled instructions.
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1771 case AArch64::STRSui:
1772 case AArch64::STRDui:
1773 case AArch64::STRQui:
1774 case AArch64::STRXui:
1775 case AArch64::STRWui:
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]1776 case AArch64::STRHHui:
1777 case AArch64::STRBBui:
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1778 case AArch64::LDRSui:
1779 case AArch64::LDRDui:
1780 case AArch64::LDRQui:
1781 case AArch64::LDRXui:
1782 case AArch64::LDRWui:
Chad Rosierdabe2532015-09-29 18:26:15 +0000[diff] [blame]1783 case AArch64::LDRHHui:
1784 case AArch64::LDRBBui:
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1785 // Unscaled instructions.
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1786 case AArch64::STURSi:
1787 case AArch64::STURDi:
1788 case AArch64::STURQi:
1789 case AArch64::STURWi:
1790 case AArch64::STURXi:
1791 case AArch64::LDURSi:
1792 case AArch64::LDURDi:
1793 case AArch64::LDURQi:
1794 case AArch64::LDURWi:
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1795 case AArch64::LDURXi:
1796 // Paired instructions.
1797 case AArch64::LDPSi:
Chad Rosier43150122015-09-29 20:39:55 +0000[diff] [blame]1798 case AArch64::LDPSWi:
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1799 case AArch64::LDPDi:
1800 case AArch64::LDPQi:
1801 case AArch64::LDPWi:
1802 case AArch64::LDPXi:
1803 case AArch64::STPSi:
1804 case AArch64::STPDi:
1805 case AArch64::STPQi:
1806 case AArch64::STPWi:
1807 case AArch64::STPXi: {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1808 // Make sure this is a reg+imm (as opposed to an address reloc).
Chad Rosierf77e9092015-08-06 15:50:12 +0000[diff] [blame]1809 if (!getLdStOffsetOp(MI).isImm()) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1810 ++MBBI;
1811 break;
1812 }
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1813 // Look forward to try to form a post-index instruction. For example,
1814 // ldr x0, [x20]
1815 // add x20, x20, #32
1816 // merged into:
1817 // ldr x0, [x20], #32
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1818 MachineBasicBlock::iterator Update =
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1819 findMatchingUpdateInsnForward(MBBI, 0, UpdateLimit);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1820 if (Update != E) {
1821 // Merge the update into the ld/st.
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]1822 MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/false);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1823 Modified = true;
1824 ++NumPostFolded;
1825 break;
1826 }
1827 // Don't know how to handle pre/post-index versions, so move to the next
1828 // instruction.
Chad Rosiere4e15ba2016-03-09 17:29:48 +0000[diff] [blame]1829 if (TII->isUnscaledLdSt(Opc)) {
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1830 ++MBBI;
1831 break;
1832 }
1833
1834 // Look back to try to find a pre-index instruction. For example,
1835 // add x0, x0, #8
1836 // ldr x1, [x0]
1837 // merged into:
1838 // ldr x1, [x0, #8]!
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1839 Update = findMatchingUpdateInsnBackward(MBBI, UpdateLimit);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1840 if (Update != E) {
1841 // Merge the update into the ld/st.
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]1842 MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1843 Modified = true;
1844 ++NumPreFolded;
1845 break;
1846 }
Chad Rosier7a83d772015-10-01 13:09:44 +0000[diff] [blame]1847 // The immediate in the load/store is scaled by the size of the memory
1848 // operation. The immediate in the add we're looking for,
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1849 // however, is not, so adjust here.
Chad Rosier0b15e7c2015-10-01 13:33:31 +0000[diff] [blame]1850 int UnscaledOffset = getLdStOffsetOp(MI).getImm() * getMemScale(MI);
Chad Rosier1bbd7fb2015-09-25 17:48:17 +0000[diff] [blame]1851
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1852 // Look forward to try to find a post-index instruction. For example,
1853 // ldr x1, [x0, #64]
1854 // add x0, x0, #64
1855 // merged into:
1856 // ldr x1, [x0, #64]!
Chad Rosier35706ad2016-02-04 21:26:02 +0000[diff] [blame]1857 Update = findMatchingUpdateInsnForward(MBBI, UnscaledOffset, UpdateLimit);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1858 if (Update != E) {
1859 // Merge the update into the ld/st.
Chad Rosier2dfd3542015-09-23 13:51:44 +0000[diff] [blame]1860 MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1861 Modified = true;
1862 ++NumPreFolded;
1863 break;
1864 }
1865
1866 // Nothing found. Just move to the next instruction.
1867 ++MBBI;
1868 break;
1869 }
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1870 }
1871 }
1872
1873 return Modified;
1874}
1875
1876bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
Andrew Kaylor1ac98bb2016-04-25 21:58:52 +0000[diff] [blame]1877 if (skipFunction(*Fn.getFunction()))
1878 return false;
1879
Oliver Stannardd414c992015-11-10 11:04:18 +0000[diff] [blame]1880 Subtarget = &static_cast<const AArch64Subtarget &>(Fn.getSubtarget());
1881 TII = static_cast<const AArch64InstrInfo *>(Subtarget->getInstrInfo());
1882 TRI = Subtarget->getRegisterInfo();
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1883
Chad Rosierbba881e2016-02-02 15:02:30 +0000[diff] [blame]1884 // Resize the modified and used register bitfield trackers. We do this once
1885 // per function and then clear the bitfield each time we optimize a load or
1886 // store.
1887 ModifiedRegs.resize(TRI->getNumRegs());
1888 UsedRegs.resize(TRI->getNumRegs());
1889
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1890 bool Modified = false;
Matthias Braun651cff42016-06-02 18:03:53 +0000[diff] [blame]1891 bool enableNarrowLdOpt =
1892 Subtarget->mergeNarrowLoads() && !Subtarget->requiresStrictAlign();
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1893 for (auto &MBB : Fn)
Jun Bum Lim22fe15e2015-11-06 16:27:47 +0000[diff] [blame]1894 Modified |= optimizeBlock(MBB, enableNarrowLdOpt);
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1895
1896 return Modified;
1897}
1898
1899// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep
1900// loads and stores near one another?
1901
Chad Rosier3f8b09d2016-02-09 19:42:19 +0000[diff] [blame]1902// FIXME: When pairing store instructions it's very possible for this pass to
1903// hoist a store with a KILL marker above another use (without a KILL marker).
1904// The resulting IR is invalid, but nothing uses the KILL markers after this
1905// pass, so it's never caused a problem in practice.
1906
Chad Rosier43f5c842015-08-05 12:40:13 +0000[diff] [blame]1907/// createAArch64LoadStoreOptimizationPass - returns an instance of the
1908/// load / store optimization pass.
Tim Northover3b0846e2014-05-24 12:50:23 +0000[diff] [blame]1909FunctionPass *llvm::createAArch64LoadStoreOptimizationPass() {
1910 return new AArch64LoadStoreOpt();
1911}