blob: c4794380f79129f9c6e99a773b7b9eea390abfac [file] [log] [blame]
Chris Lattnerf2836d12007-03-31 04:06:36 +00001//===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattnerf3ebc3f2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Chris Lattnerf2836d12007-03-31 04:06:36 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This pass munges the code in the input function to better prepare it for
Gordon Henriksen829046b2008-05-08 17:46:35 +000011// SelectionDAG-based code generation. This works around limitations in it's
12// basic-block-at-a-time approach. It should eventually be removed.
Chris Lattnerf2836d12007-03-31 04:06:36 +000013//
14//===----------------------------------------------------------------------===//
15
Eugene Zelenko900b6332017-08-29 22:32:07 +000016#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000018#include "llvm/ADT/DenseMap.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000019#include "llvm/ADT/PointerIntPair.h"
20#include "llvm/ADT/STLExtras.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000021#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallVector.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000023#include "llvm/ADT/Statistic.h"
Jun Bum Lim90b6b502016-12-16 20:38:39 +000024#include "llvm/Analysis/BlockFrequencyInfo.h"
25#include "llvm/Analysis/BranchProbabilityInfo.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000026#include "llvm/Analysis/ConstantFolding.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000027#include "llvm/Analysis/InstructionSimplify.h"
Chuang-Yu Chengd3fb38c2016-04-05 14:06:20 +000028#include "llvm/Analysis/LoopInfo.h"
Zaara Syeda3a7578c2017-05-31 17:12:38 +000029#include "llvm/Analysis/MemoryBuiltins.h"
Dehao Chen302b69c2016-10-18 20:42:47 +000030#include "llvm/Analysis/ProfileSummaryInfo.h"
Chandler Carruth62d42152015-01-15 02:16:27 +000031#include "llvm/Analysis/TargetLibraryInfo.h"
Quentin Colombetc32615d2014-10-31 17:52:53 +000032#include "llvm/Analysis/TargetTransformInfo.h"
Sanjay Patel69a50a12015-10-19 21:59:12 +000033#include "llvm/Analysis/ValueTracking.h"
Michael Kupersteinf79af6f2016-09-08 00:48:37 +000034#include "llvm/CodeGen/Analysis.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000035#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineValueType.h"
37#include "llvm/CodeGen/SelectionDAGNodes.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000038#include "llvm/CodeGen/TargetLowering.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000039#include "llvm/CodeGen/TargetPassConfig.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000040#include "llvm/CodeGen/TargetSubtargetInfo.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000041#include "llvm/CodeGen/ValueTypes.h"
42#include "llvm/IR/Argument.h"
43#include "llvm/IR/Attributes.h"
44#include "llvm/IR/BasicBlock.h"
Chandler Carruth219b89b2014-03-04 11:01:28 +000045#include "llvm/IR/CallSite.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000046#include "llvm/IR/Constant.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000047#include "llvm/IR/Constants.h"
48#include "llvm/IR/DataLayout.h"
49#include "llvm/IR/DerivedTypes.h"
Chandler Carruth5ad5f152014-01-13 09:26:24 +000050#include "llvm/IR/Dominators.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000051#include "llvm/IR/Function.h"
Chandler Carruth03eb0de2014-03-04 10:40:04 +000052#include "llvm/IR/GetElementPtrTypeIterator.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000053#include "llvm/IR/GlobalValue.h"
54#include "llvm/IR/GlobalVariable.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000055#include "llvm/IR/IRBuilder.h"
56#include "llvm/IR/InlineAsm.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000057#include "llvm/IR/InstrTypes.h"
58#include "llvm/IR/Instruction.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000059#include "llvm/IR/Instructions.h"
60#include "llvm/IR/IntrinsicInst.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000061#include "llvm/IR/Intrinsics.h"
62#include "llvm/IR/LLVMContext.h"
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +000063#include "llvm/IR/MDBuilder.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000064#include "llvm/IR/Module.h"
65#include "llvm/IR/Operator.h"
Chandler Carruth820a9082014-03-04 11:08:18 +000066#include "llvm/IR/PatternMatch.h"
Ramkumar Ramachandradba73292015-01-14 23:27:07 +000067#include "llvm/IR/Statepoint.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000068#include "llvm/IR/Type.h"
69#include "llvm/IR/Use.h"
70#include "llvm/IR/User.h"
71#include "llvm/IR/Value.h"
Chandler Carruth4220e9c2014-03-04 11:17:44 +000072#include "llvm/IR/ValueHandle.h"
Chandler Carrutha4ea2692014-03-04 11:26:31 +000073#include "llvm/IR/ValueMap.h"
Chris Lattnerf2836d12007-03-31 04:06:36 +000074#include "llvm/Pass.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000075#include "llvm/Support/BlockFrequency.h"
Sanjay Pateld66607b2016-04-26 17:11:17 +000076#include "llvm/Support/BranchProbability.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000077#include "llvm/Support/Casting.h"
Evan Cheng8b637b12010-08-17 01:34:49 +000078#include "llvm/Support/CommandLine.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000079#include "llvm/Support/Compiler.h"
Evan Chengd3d80172007-12-05 23:58:20 +000080#include "llvm/Support/Debug.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000081#include "llvm/Support/ErrorHandling.h"
82#include "llvm/Support/MathExtras.h"
Chandler Carruthaafe0912012-06-29 12:38:19 +000083#include "llvm/Support/raw_ostream.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000084#include "llvm/Target/TargetMachine.h"
85#include "llvm/Target/TargetOptions.h"
Chandler Carruthaafe0912012-06-29 12:38:19 +000086#include "llvm/Transforms/Utils/BasicBlockUtils.h"
Preston Gurdcdf540d2012-09-04 18:22:17 +000087#include "llvm/Transforms/Utils/BypassSlowDivision.h"
Chandler Carruthaafe0912012-06-29 12:38:19 +000088#include "llvm/Transforms/Utils/Local.h"
Ahmed Bougachae03bef72015-01-12 17:22:43 +000089#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
Eugene Zelenko900b6332017-08-29 22:32:07 +000090#include <algorithm>
91#include <cassert>
92#include <cstdint>
93#include <iterator>
94#include <limits>
95#include <memory>
96#include <utility>
97#include <vector>
Zaara Syeda3a7578c2017-05-31 17:12:38 +000098
Chris Lattnerf2836d12007-03-31 04:06:36 +000099using namespace llvm;
Chris Lattnerd616ef52008-11-25 04:42:10 +0000100using namespace llvm::PatternMatch;
Chris Lattnerf2836d12007-03-31 04:06:36 +0000101
Chandler Carruth1b9dde02014-04-22 02:02:50 +0000102#define DEBUG_TYPE "codegenprepare"
103
Cameron Zwarichced753f2011-01-05 17:27:27 +0000104STATISTIC(NumBlocksElim, "Number of blocks eliminated");
Evan Cheng0663f232011-03-21 01:19:09 +0000105STATISTIC(NumPHIsElim, "Number of trivial PHIs eliminated");
106STATISTIC(NumGEPsElim, "Number of GEPs converted to casts");
Cameron Zwarichced753f2011-01-05 17:27:27 +0000107STATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of "
108 "sunken Cmps");
109STATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses "
110 "of sunken Casts");
111STATISTIC(NumMemoryInsts, "Number of memory instructions whose address "
112 "computations were sunk");
Serguei Katkovd5d8d542017-11-05 05:50:33 +0000113STATISTIC(NumMemoryInstsPhiCreated,
114 "Number of phis created when address "
115 "computations were sunk to memory instructions");
116STATISTIC(NumMemoryInstsSelectCreated,
117 "Number of select created when address "
118 "computations were sunk to memory instructions");
Evan Cheng0663f232011-03-21 01:19:09 +0000119STATISTIC(NumExtsMoved, "Number of [s|z]ext instructions combined with loads");
120STATISTIC(NumExtUses, "Number of uses of [s|z]ext instructions optimized");
Geoff Berry5256fca2015-11-20 22:34:39 +0000121STATISTIC(NumAndsAdded,
122 "Number of and mask instructions added to form ext loads");
123STATISTIC(NumAndUses, "Number of uses of and mask instructions optimized");
Evan Cheng0663f232011-03-21 01:19:09 +0000124STATISTIC(NumRetsDup, "Number of return instructions duplicated");
Devang Patel53771ba2011-08-18 00:50:51 +0000125STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");
Benjamin Kramer047d7ca2012-05-05 12:49:22 +0000126STATISTIC(NumSelectsExpanded, "Number of selects turned into branches");
Quentin Colombetc32615d2014-10-31 17:52:53 +0000127STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed");
Jakob Stoklund Oleseneb12f492010-09-30 20:51:52 +0000128
Cameron Zwarich338d3622011-03-11 21:52:04 +0000129static cl::opt<bool> DisableBranchOpts(
130 "disable-cgp-branch-opts", cl::Hidden, cl::init(false),
131 cl::desc("Disable branch optimizations in CodeGenPrepare"));
132
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000133static cl::opt<bool>
134 DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false),
135 cl::desc("Disable GC optimizations in CodeGenPrepare"));
136
Benjamin Kramer3d38c172012-05-06 14:25:16 +0000137static cl::opt<bool> DisableSelectToBranch(
138 "disable-cgp-select2branch", cl::Hidden, cl::init(false),
139 cl::desc("Disable select to branch conversion."));
Benjamin Kramer047d7ca2012-05-05 12:49:22 +0000140
Hal Finkelc3998302014-04-12 00:59:48 +0000141static cl::opt<bool> AddrSinkUsingGEPs(
Eli Friedman5fba1e52017-04-06 22:42:18 +0000142 "addr-sink-using-gep", cl::Hidden, cl::init(true),
Hal Finkelc3998302014-04-12 00:59:48 +0000143 cl::desc("Address sinking in CGP using GEPs."));
144
Tim Northovercea0abb2014-03-29 08:22:29 +0000145static cl::opt<bool> EnableAndCmpSinking(
146 "enable-andcmp-sinking", cl::Hidden, cl::init(true),
147 cl::desc("Enable sinkinig and/cmp into branches."));
148
Quentin Colombetc32615d2014-10-31 17:52:53 +0000149static cl::opt<bool> DisableStoreExtract(
150 "disable-cgp-store-extract", cl::Hidden, cl::init(false),
151 cl::desc("Disable store(extract) optimizations in CodeGenPrepare"));
152
153static cl::opt<bool> StressStoreExtract(
154 "stress-cgp-store-extract", cl::Hidden, cl::init(false),
155 cl::desc("Stress test store(extract) optimizations in CodeGenPrepare"));
156
Quentin Colombetfc2201e2014-12-17 01:36:17 +0000157static cl::opt<bool> DisableExtLdPromotion(
158 "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
159 cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
160 "CodeGenPrepare"));
161
162static cl::opt<bool> StressExtLdPromotion(
163 "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
164 cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
165 "optimization in CodeGenPrepare"));
166
Chuang-Yu Chengd3fb38c2016-04-05 14:06:20 +0000167static cl::opt<bool> DisablePreheaderProtect(
168 "disable-preheader-prot", cl::Hidden, cl::init(false),
169 cl::desc("Disable protection against removing loop preheaders"));
170
Dehao Chen302b69c2016-10-18 20:42:47 +0000171static cl::opt<bool> ProfileGuidedSectionPrefix(
David Callahan5960d9b12017-06-14 20:35:33 +0000172 "profile-guided-section-prefix", cl::Hidden, cl::init(true), cl::ZeroOrMore,
Dehao Chen302b69c2016-10-18 20:42:47 +0000173 cl::desc("Use profile info to add section prefix for hot/cold functions"));
174
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000175static cl::opt<unsigned> FreqRatioToSkipMerge(
176 "cgp-freq-ratio-to-skip-merge", cl::Hidden, cl::init(2),
177 cl::desc("Skip merging empty blocks if (frequency of empty block) / "
178 "(frequency of destination block) is greater than this ratio"));
179
Wei Mia2f0b592016-12-22 19:44:45 +0000180static cl::opt<bool> ForceSplitStore(
181 "force-split-store", cl::Hidden, cl::init(false),
182 cl::desc("Force store splitting no matter what the target query says."));
183
Jun Bum Limdee55652017-04-03 19:20:07 +0000184static cl::opt<bool>
185EnableTypePromotionMerge("cgp-type-promotion-merge", cl::Hidden,
186 cl::desc("Enable merging of redundant sexts when one is dominating"
187 " the other."), cl::init(true));
188
Serguei Katkovd5d8d542017-11-05 05:50:33 +0000189static cl::opt<bool> DisableComplexAddrModes(
Serguei Katkovd4df7442017-11-29 09:48:50 +0000190 "disable-complex-addr-modes", cl::Hidden, cl::init(false),
Serguei Katkovd5d8d542017-11-05 05:50:33 +0000191 cl::desc("Disables combining addressing modes with different parts "
192 "in optimizeMemoryInst."));
193
194static cl::opt<bool>
195AddrSinkNewPhis("addr-sink-new-phis", cl::Hidden, cl::init(false),
196 cl::desc("Allow creation of Phis in Address sinking."));
197
198static cl::opt<bool>
Serguei Katkovac4a8fb2017-12-13 07:39:35 +0000199AddrSinkNewSelects("addr-sink-new-select", cl::Hidden, cl::init(false),
Serguei Katkovd5d8d542017-11-05 05:50:33 +0000200 cl::desc("Allow creation of selects in Address sinking."));
201
John Brawn70cdb5b2017-11-24 14:10:45 +0000202static cl::opt<bool> AddrSinkCombineBaseReg(
203 "addr-sink-combine-base-reg", cl::Hidden, cl::init(true),
204 cl::desc("Allow combining of BaseReg field in Address sinking."));
205
206static cl::opt<bool> AddrSinkCombineBaseGV(
207 "addr-sink-combine-base-gv", cl::Hidden, cl::init(true),
208 cl::desc("Allow combining of BaseGV field in Address sinking."));
209
210static cl::opt<bool> AddrSinkCombineBaseOffs(
211 "addr-sink-combine-base-offs", cl::Hidden, cl::init(true),
212 cl::desc("Allow combining of BaseOffs field in Address sinking."));
213
214static cl::opt<bool> AddrSinkCombineScaledReg(
215 "addr-sink-combine-scaled-reg", cl::Hidden, cl::init(true),
216 cl::desc("Allow combining of ScaledReg field in Address sinking."));
217
Eric Christopherc1ea1492008-09-24 05:32:41 +0000218namespace {
Eugene Zelenko900b6332017-08-29 22:32:07 +0000219
220using SetOfInstrs = SmallPtrSet<Instruction *, 16>;
221using TypeIsSExt = PointerIntPair<Type *, 1, bool>;
222using InstrToOrigTy = DenseMap<Instruction *, TypeIsSExt>;
223using SExts = SmallVector<Instruction *, 16>;
224using ValueToSExts = DenseMap<Value *, SExts>;
225
Quentin Colombetfc2201e2014-12-17 01:36:17 +0000226class TypePromotionTransaction;
Quentin Colombet3a4bf042014-02-06 21:44:56 +0000227
Chris Lattner2dd09db2009-09-02 06:11:42 +0000228 class CodeGenPrepare : public FunctionPass {
Eugene Zelenko900b6332017-08-29 22:32:07 +0000229 const TargetMachine *TM = nullptr;
Igor Laevsky3be81ba2017-02-07 13:27:20 +0000230 const TargetSubtargetInfo *SubtargetInfo;
Eugene Zelenko900b6332017-08-29 22:32:07 +0000231 const TargetLowering *TLI = nullptr;
Igor Laevsky3be81ba2017-02-07 13:27:20 +0000232 const TargetRegisterInfo *TRI;
Eugene Zelenko900b6332017-08-29 22:32:07 +0000233 const TargetTransformInfo *TTI = nullptr;
Chad Rosierc24b86f2011-12-01 03:08:23 +0000234 const TargetLibraryInfo *TLInfo;
Chuang-Yu Chengd3fb38c2016-04-05 14:06:20 +0000235 const LoopInfo *LI;
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000236 std::unique_ptr<BlockFrequencyInfo> BFI;
237 std::unique_ptr<BranchProbabilityInfo> BPI;
Nadav Rotem465834c2012-07-24 10:51:42 +0000238
Sanjay Patel4ac6b112015-09-21 22:47:23 +0000239 /// As we scan instructions optimizing them, this is the next instruction
240 /// to optimize. Transforms that can invalidate this should update it.
Chris Lattner7a277142011-01-15 07:14:54 +0000241 BasicBlock::iterator CurInstIterator;
Evan Cheng3b3de7c2008-12-19 18:03:11 +0000242
Evan Cheng0663f232011-03-21 01:19:09 +0000243 /// Keeps track of non-local addresses that have been sunk into a block.
244 /// This allows us to avoid inserting duplicate code for blocks with
Simon Dardis230f4532017-11-24 16:45:28 +0000245 /// multiple load/stores of the same address. The usage of WeakTrackingVH
246 /// enables SunkAddrs to be treated as a cache whose entries can be
247 /// invalidated if a sunken address computation has been erased.
248 ValueMap<Value*, WeakTrackingVH> SunkAddrs;
Cameron Zwarichce3b9302011-01-06 00:42:50 +0000249
Ahmed Bougachaf3299142015-06-17 20:44:32 +0000250 /// Keeps track of all instructions inserted for the current function.
251 SetOfInstrs InsertedInsts;
Eugene Zelenko900b6332017-08-29 22:32:07 +0000252
Quentin Colombet3a4bf042014-02-06 21:44:56 +0000253 /// Keeps track of the type of the related instruction before their
254 /// promotion for the current function.
255 InstrToOrigTy PromotedInsts;
256
Jun Bum Limdee55652017-04-03 19:20:07 +0000257 /// Keep track of instructions removed during promotion.
258 SetOfInstrs RemovedInsts;
259
260 /// Keep track of sext chains based on their initial value.
261 DenseMap<Value *, Instruction *> SeenChainsForSExt;
262
263 /// Keep track of SExt promoted.
264 ValueToSExts ValToSExtendedUses;
265
Sanjay Patel4ac6b112015-09-21 22:47:23 +0000266 /// True if CFG is modified in any way.
Devang Patel8f606d72011-03-24 15:35:25 +0000267 bool ModifiedDT;
Evan Cheng0663f232011-03-21 01:19:09 +0000268
Sanjay Patel4ac6b112015-09-21 22:47:23 +0000269 /// True if optimizing for size.
Benjamin Kramer047d7ca2012-05-05 12:49:22 +0000270 bool OptSize;
271
Mehdi Amini4fe37982015-07-07 18:45:17 +0000272 /// DataLayout for the Function being processed.
Eugene Zelenko900b6332017-08-29 22:32:07 +0000273 const DataLayout *DL = nullptr;
Mehdi Amini4fe37982015-07-07 18:45:17 +0000274
Chris Lattnerf2836d12007-03-31 04:06:36 +0000275 public:
Nick Lewyckye7da2d62007-05-06 13:37:16 +0000276 static char ID; // Pass identification, replacement for typeid
Eugene Zelenko900b6332017-08-29 22:32:07 +0000277
278 CodeGenPrepare() : FunctionPass(ID) {
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000279 initializeCodeGenPreparePass(*PassRegistry::getPassRegistry());
280 }
Eugene Zelenko900b6332017-08-29 22:32:07 +0000281
Craig Topper4584cd52014-03-07 09:26:03 +0000282 bool runOnFunction(Function &F) override;
Eric Christopherc1ea1492008-09-24 05:32:41 +0000283
Mehdi Amini117296c2016-10-01 02:56:57 +0000284 StringRef getPassName() const override { return "CodeGen Prepare"; }
Evan Cheng99cafb12012-12-21 01:48:14 +0000285
Craig Topper4584cd52014-03-07 09:26:03 +0000286 void getAnalysisUsage(AnalysisUsage &AU) const override {
George Burgess IVd4febd12016-03-22 21:25:08 +0000287 // FIXME: When we can selectively preserve passes, preserve the domtree.
Dehao Chen302b69c2016-10-18 20:42:47 +0000288 AU.addRequired<ProfileSummaryInfoWrapperPass>();
Chandler Carruthb98f63d2015-01-15 10:41:28 +0000289 AU.addRequired<TargetLibraryInfoWrapperPass>();
Chandler Carruth705b1852015-01-31 03:43:40 +0000290 AU.addRequired<TargetTransformInfoWrapperPass>();
Chuang-Yu Chengd3fb38c2016-04-05 14:06:20 +0000291 AU.addRequired<LoopInfoWrapperPass>();
Andreas Neustifterf8cb7582009-09-16 09:26:52 +0000292 }
293
Chris Lattnerf2836d12007-03-31 04:06:36 +0000294 private:
Sanjay Patelfc580a62015-09-21 23:03:16 +0000295 bool eliminateFallThrough(Function &F);
296 bool eliminateMostlyEmptyBlocks(Function &F);
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000297 BasicBlock *findDestBlockOfMergeableEmptyBlock(BasicBlock *BB);
Sanjay Patelfc580a62015-09-21 23:03:16 +0000298 bool canMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const;
299 void eliminateMostlyEmptyBlock(BasicBlock *BB);
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000300 bool isMergingEmptyBlockProfitable(BasicBlock *BB, BasicBlock *DestBB,
301 bool isPreheader);
Sanjay Patel3b8974b2017-06-08 20:00:09 +0000302 bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT);
303 bool optimizeInst(Instruction *I, bool &ModifiedDT);
Sanjay Patelfc580a62015-09-21 23:03:16 +0000304 bool optimizeMemoryInst(Instruction *I, Value *Addr,
Matt Arsenaultf72b49b2015-06-04 16:17:38 +0000305 Type *AccessTy, unsigned AS);
Sanjay Patelfc580a62015-09-21 23:03:16 +0000306 bool optimizeInlineAsmInst(CallInst *CS);
Sanjay Patel3b8974b2017-06-08 20:00:09 +0000307 bool optimizeCallInst(CallInst *CI, bool &ModifiedDT);
Jun Bum Limdee55652017-04-03 19:20:07 +0000308 bool optimizeExt(Instruction *&I);
Sanjay Patelfc580a62015-09-21 23:03:16 +0000309 bool optimizeExtUses(Instruction *I);
Geoff Berry5256fca2015-11-20 22:34:39 +0000310 bool optimizeLoadExt(LoadInst *I);
Sanjay Patelfc580a62015-09-21 23:03:16 +0000311 bool optimizeSelectInst(SelectInst *SI);
312 bool optimizeShuffleVectorInst(ShuffleVectorInst *SI);
Sanjay Patel0ed9aea2015-11-02 23:22:49 +0000313 bool optimizeSwitchInst(SwitchInst *CI);
Sanjay Patelfc580a62015-09-21 23:03:16 +0000314 bool optimizeExtractElementInst(Instruction *Inst);
315 bool dupRetToEnableTailCallOpts(BasicBlock *BB);
316 bool placeDbgValues(Function &F);
Jun Bum Lim42301012017-03-17 19:05:21 +0000317 bool canFormExtLd(const SmallVectorImpl<Instruction *> &MovedExts,
318 LoadInst *&LI, Instruction *&Inst, bool HasPromoted);
319 bool tryToPromoteExts(TypePromotionTransaction &TPT,
320 const SmallVectorImpl<Instruction *> &Exts,
321 SmallVectorImpl<Instruction *> &ProfitablyMovedExts,
322 unsigned CreatedInstsCost = 0);
Jun Bum Limdee55652017-04-03 19:20:07 +0000323 bool mergeSExts(Function &F);
324 bool performAddressTypePromotion(
325 Instruction *&Inst,
326 bool AllowPromotionWithoutCommonHeader,
327 bool HasPromoted, TypePromotionTransaction &TPT,
328 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts);
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +0000329 bool splitBranchCondition(Function &F);
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000330 bool simplifyOffsetableRelocate(Instruction &I);
Chris Lattnerf2836d12007-03-31 04:06:36 +0000331 };
Eugene Zelenko900b6332017-08-29 22:32:07 +0000332
333} // end anonymous namespace
Devang Patel09f162c2007-05-01 21:15:47 +0000334
Devang Patel8c78a0b2007-05-03 01:11:54 +0000335char CodeGenPrepare::ID = 0;
Eugene Zelenko900b6332017-08-29 22:32:07 +0000336
Matthias Braun1527baa2017-05-25 21:26:32 +0000337INITIALIZE_PASS_BEGIN(CodeGenPrepare, DEBUG_TYPE,
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000338 "Optimize for code generation", false, false)
Dehao Chen302b69c2016-10-18 20:42:47 +0000339INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
Matthias Braun1527baa2017-05-25 21:26:32 +0000340INITIALIZE_PASS_END(CodeGenPrepare, DEBUG_TYPE,
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000341 "Optimize for code generation", false, false)
Chris Lattnerf2836d12007-03-31 04:06:36 +0000342
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000343FunctionPass *llvm::createCodeGenPreparePass() { return new CodeGenPrepare(); }
Chris Lattnerf2836d12007-03-31 04:06:36 +0000344
Chris Lattnerf2836d12007-03-31 04:06:36 +0000345bool CodeGenPrepare::runOnFunction(Function &F) {
Andrew Kayloraa641a52016-04-22 22:06:11 +0000346 if (skipFunction(F))
Paul Robinson7c99ec52014-03-31 17:43:35 +0000347 return false;
348
Mehdi Amini4fe37982015-07-07 18:45:17 +0000349 DL = &F.getParent()->getDataLayout();
350
Chris Lattnerf2836d12007-03-31 04:06:36 +0000351 bool EverMadeChange = false;
Quentin Colombet3a4bf042014-02-06 21:44:56 +0000352 // Clear per function information.
Ahmed Bougachaf3299142015-06-17 20:44:32 +0000353 InsertedInsts.clear();
Quentin Colombet3a4bf042014-02-06 21:44:56 +0000354 PromotedInsts.clear();
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000355 BFI.reset();
356 BPI.reset();
Eric Christopherc1ea1492008-09-24 05:32:41 +0000357
Devang Patel8f606d72011-03-24 15:35:25 +0000358 ModifiedDT = false;
Francis Visoiu Mistrih8b617642017-05-18 17:21:13 +0000359 if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>()) {
360 TM = &TPC->getTM<TargetMachine>();
Igor Laevsky3be81ba2017-02-07 13:27:20 +0000361 SubtargetInfo = TM->getSubtargetImpl(F);
362 TLI = SubtargetInfo->getTargetLowering();
363 TRI = SubtargetInfo->getRegisterInfo();
364 }
Chandler Carruthb98f63d2015-01-15 10:41:28 +0000365 TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
Chandler Carruthfdb9c572015-02-01 12:01:35 +0000366 TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
Chuang-Yu Chengd3fb38c2016-04-05 14:06:20 +0000367 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
Sanjay Patel82d91dd2015-08-11 19:39:36 +0000368 OptSize = F.optForSize();
Evan Cheng0663f232011-03-21 01:19:09 +0000369
Easwaran Raman0d55b552017-11-14 19:31:51 +0000370 ProfileSummaryInfo *PSI =
371 getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
Dehao Chen302b69c2016-10-18 20:42:47 +0000372 if (ProfileGuidedSectionPrefix) {
Dehao Chen775341a2017-03-23 23:14:11 +0000373 if (PSI->isFunctionHotInCallGraph(&F))
Dehao Chen302b69c2016-10-18 20:42:47 +0000374 F.setSectionPrefix(".hot");
Dehao Chen775341a2017-03-23 23:14:11 +0000375 else if (PSI->isFunctionColdInCallGraph(&F))
Teresa Johnson720d9b42017-05-09 01:43:24 +0000376 F.setSectionPrefix(".unlikely");
Dehao Chen302b69c2016-10-18 20:42:47 +0000377 }
378
Preston Gurdcdf540d2012-09-04 18:22:17 +0000379 /// This optimization identifies DIV instructions that can be
380 /// profitably bypassed and carried out with a shorter, faster divide.
Easwaran Raman0d55b552017-11-14 19:31:51 +0000381 if (!OptSize && !PSI->hasHugeWorkingSetSize() && TLI &&
382 TLI->isSlowDivBypassed()) {
Preston Gurd0d67f512012-10-04 21:33:40 +0000383 const DenseMap<unsigned int, unsigned int> &BypassWidths =
384 TLI->getBypassSlowDivWidths();
Eric Christopher49a7d6c2016-01-04 23:18:58 +0000385 BasicBlock* BB = &*F.begin();
386 while (BB != nullptr) {
387 // bypassSlowDivision may create new BBs, but we don't want to reapply the
388 // optimization to those blocks.
389 BasicBlock* Next = BB->getNextNode();
390 EverMadeChange |= bypassSlowDivision(BB, BypassWidths);
391 BB = Next;
392 }
Preston Gurdcdf540d2012-09-04 18:22:17 +0000393 }
394
395 // Eliminate blocks that contain only PHI nodes and an
Chris Lattnerc3748562007-04-02 01:35:34 +0000396 // unconditional branch.
Sanjay Patelfc580a62015-09-21 23:03:16 +0000397 EverMadeChange |= eliminateMostlyEmptyBlocks(F);
Eric Christopherc1ea1492008-09-24 05:32:41 +0000398
Devang Patel53771ba2011-08-18 00:50:51 +0000399 // llvm.dbg.value is far away from the value then iSel may not be able
Nadav Rotem465834c2012-07-24 10:51:42 +0000400 // handle it properly. iSel will drop llvm.dbg.value if it can not
Devang Patel53771ba2011-08-18 00:50:51 +0000401 // find a node corresponding to the value.
Sanjay Patelfc580a62015-09-21 23:03:16 +0000402 EverMadeChange |= placeDbgValues(F);
Devang Patel53771ba2011-08-18 00:50:51 +0000403
Geoff Berry5d534b62017-02-21 18:53:14 +0000404 if (!DisableBranchOpts)
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +0000405 EverMadeChange |= splitBranchCondition(F);
Tim Northovercea0abb2014-03-29 08:22:29 +0000406
Michael Kuperstein13bf8a22017-02-28 00:11:34 +0000407 // Split some critical edges where one of the sources is an indirect branch,
408 // to help generate sane code for PHIs involving such edges.
Hiroshi Yamauchi9364fa32017-12-04 20:36:01 +0000409 EverMadeChange |= SplitIndirectBrCriticalEdges(F);
Michael Kuperstein13bf8a22017-02-28 00:11:34 +0000410
Chris Lattnerc3748562007-04-02 01:35:34 +0000411 bool MadeChange = true;
Chris Lattnerf2836d12007-03-31 04:06:36 +0000412 while (MadeChange) {
413 MadeChange = false;
Jun Bum Limdee55652017-04-03 19:20:07 +0000414 SeenChainsForSExt.clear();
415 ValToSExtendedUses.clear();
416 RemovedInsts.clear();
Hans Wennborg02fbc712012-09-19 07:48:16 +0000417 for (Function::iterator I = F.begin(); I != F.end(); ) {
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +0000418 BasicBlock *BB = &*I++;
Elena Demikhovsky87700a72014-12-28 08:54:45 +0000419 bool ModifiedDTOnIteration = false;
Sanjay Patelfc580a62015-09-21 23:03:16 +0000420 MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration);
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000421
Elena Demikhovsky87700a72014-12-28 08:54:45 +0000422 // Restart BB iteration if the dominator tree of the Function was changed
Elena Demikhovsky87700a72014-12-28 08:54:45 +0000423 if (ModifiedDTOnIteration)
424 break;
Evan Cheng0663f232011-03-21 01:19:09 +0000425 }
Jun Bum Limdee55652017-04-03 19:20:07 +0000426 if (EnableTypePromotionMerge && !ValToSExtendedUses.empty())
427 MadeChange |= mergeSExts(F);
428
429 // Really free removed instructions during promotion.
430 for (Instruction *I : RemovedInsts)
Reid Kleckner96ab8722017-05-18 17:24:10 +0000431 I->deleteValue();
Jun Bum Limdee55652017-04-03 19:20:07 +0000432
Chris Lattnerf2836d12007-03-31 04:06:36 +0000433 EverMadeChange |= MadeChange;
434 }
Cameron Zwarichce3b9302011-01-06 00:42:50 +0000435
436 SunkAddrs.clear();
437
Cameron Zwarich338d3622011-03-11 21:52:04 +0000438 if (!DisableBranchOpts) {
439 MadeChange = false;
Bill Wendling97b93592012-03-04 10:46:01 +0000440 SmallPtrSet<BasicBlock*, 8> WorkList;
Duncan P. N. Exon Smith5914a972015-01-08 20:44:33 +0000441 for (BasicBlock &BB : F) {
442 SmallVector<BasicBlock *, 2> Successors(succ_begin(&BB), succ_end(&BB));
443 MadeChange |= ConstantFoldTerminator(&BB, true);
Bill Wendling97b93592012-03-04 10:46:01 +0000444 if (!MadeChange) continue;
445
446 for (SmallVectorImpl<BasicBlock*>::iterator
447 II = Successors.begin(), IE = Successors.end(); II != IE; ++II)
448 if (pred_begin(*II) == pred_end(*II))
449 WorkList.insert(*II);
450 }
451
Bill Wendlingf3614fd2012-11-28 23:23:48 +0000452 // Delete the dead blocks and any of their dead successors.
Bill Wendlingab417b62012-12-06 00:30:20 +0000453 MadeChange |= !WorkList.empty();
Bill Wendlingf3614fd2012-11-28 23:23:48 +0000454 while (!WorkList.empty()) {
455 BasicBlock *BB = *WorkList.begin();
456 WorkList.erase(BB);
457 SmallVector<BasicBlock*, 2> Successors(succ_begin(BB), succ_end(BB));
458
459 DeleteDeadBlock(BB);
Stephen Lin837bba12013-07-15 17:55:02 +0000460
Bill Wendlingf3614fd2012-11-28 23:23:48 +0000461 for (SmallVectorImpl<BasicBlock*>::iterator
462 II = Successors.begin(), IE = Successors.end(); II != IE; ++II)
463 if (pred_begin(*II) == pred_end(*II))
464 WorkList.insert(*II);
465 }
Cameron Zwarich338d3622011-03-11 21:52:04 +0000466
Nadav Rotem70409992012-08-14 05:19:07 +0000467 // Merge pairs of basic blocks with unconditional branches, connected by
468 // a single edge.
469 if (EverMadeChange || MadeChange)
Sanjay Patelfc580a62015-09-21 23:03:16 +0000470 MadeChange |= eliminateFallThrough(F);
Nadav Rotem70409992012-08-14 05:19:07 +0000471
Cameron Zwarich338d3622011-03-11 21:52:04 +0000472 EverMadeChange |= MadeChange;
473 }
474
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000475 if (!DisableGCOpts) {
476 SmallVector<Instruction *, 2> Statepoints;
477 for (BasicBlock &BB : F)
478 for (Instruction &I : BB)
479 if (isStatepoint(I))
480 Statepoints.push_back(&I);
481 for (auto &I : Statepoints)
482 EverMadeChange |= simplifyOffsetableRelocate(*I);
483 }
484
Chris Lattnerf2836d12007-03-31 04:06:36 +0000485 return EverMadeChange;
486}
487
Sanjay Patel4ac6b112015-09-21 22:47:23 +0000488/// Merge basic blocks which are connected by a single edge, where one of the
489/// basic blocks has a single successor pointing to the other basic block,
490/// which has a single predecessor.
Sanjay Patelfc580a62015-09-21 23:03:16 +0000491bool CodeGenPrepare::eliminateFallThrough(Function &F) {
Nadav Rotem70409992012-08-14 05:19:07 +0000492 bool Changed = false;
493 // Scan all of the blocks in the function, except for the entry block.
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +0000494 for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) {
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +0000495 BasicBlock *BB = &*I++;
Nadav Rotem70409992012-08-14 05:19:07 +0000496 // If the destination block has a single pred, then this is a trivial
497 // edge, just collapse it.
498 BasicBlock *SinglePred = BB->getSinglePredecessor();
499
Evan Cheng64a223a2012-09-28 23:58:57 +0000500 // Don't merge if BB's address is taken.
501 if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue;
Nadav Rotem70409992012-08-14 05:19:07 +0000502
503 BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator());
504 if (Term && !Term->isConditional()) {
505 Changed = true;
Michael Liao6e12d122012-08-21 05:55:22 +0000506 DEBUG(dbgs() << "To merge:\n"<< *SinglePred << "\n\n\n");
Nadav Rotem70409992012-08-14 05:19:07 +0000507 // Remember if SinglePred was the entry block of the function.
508 // If so, we will need to move BB back to the entry position.
509 bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock();
Quentin Colombet7bdd50d2015-03-18 23:17:28 +0000510 MergeBasicBlockIntoOnlyPred(BB, nullptr);
Nadav Rotem70409992012-08-14 05:19:07 +0000511
512 if (isEntry && BB != &BB->getParent()->getEntryBlock())
513 BB->moveBefore(&BB->getParent()->getEntryBlock());
514
515 // We have erased a block. Update the iterator.
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +0000516 I = BB->getIterator();
Nadav Rotem70409992012-08-14 05:19:07 +0000517 }
518 }
519 return Changed;
520}
521
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000522/// Find a destination block from BB if BB is mergeable empty block.
523BasicBlock *CodeGenPrepare::findDestBlockOfMergeableEmptyBlock(BasicBlock *BB) {
524 // If this block doesn't end with an uncond branch, ignore it.
525 BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator());
526 if (!BI || !BI->isUnconditional())
527 return nullptr;
528
529 // If the instruction before the branch (skipping debug info) isn't a phi
530 // node, then other stuff is happening here.
531 BasicBlock::iterator BBI = BI->getIterator();
532 if (BBI != BB->begin()) {
533 --BBI;
534 while (isa<DbgInfoIntrinsic>(BBI)) {
535 if (BBI == BB->begin())
536 break;
537 --BBI;
538 }
539 if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI))
540 return nullptr;
541 }
542
543 // Do not break infinite loops.
544 BasicBlock *DestBB = BI->getSuccessor(0);
545 if (DestBB == BB)
546 return nullptr;
547
548 if (!canMergeBlocks(BB, DestBB))
549 DestBB = nullptr;
550
551 return DestBB;
552}
553
Sanjay Patel4ac6b112015-09-21 22:47:23 +0000554/// Eliminate blocks that contain only PHI nodes, debug info directives, and an
555/// unconditional branch. Passes before isel (e.g. LSR/loopsimplify) often split
556/// edges in ways that are non-optimal for isel. Start by eliminating these
557/// blocks so we can split them the way we want them.
Sanjay Patelfc580a62015-09-21 23:03:16 +0000558bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F) {
Chuang-Yu Chengd3fb38c2016-04-05 14:06:20 +0000559 SmallPtrSet<BasicBlock *, 16> Preheaders;
560 SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end());
561 while (!LoopList.empty()) {
562 Loop *L = LoopList.pop_back_val();
563 LoopList.insert(LoopList.end(), L->begin(), L->end());
564 if (BasicBlock *Preheader = L->getLoopPreheader())
565 Preheaders.insert(Preheader);
566 }
567
Chris Lattnerc3748562007-04-02 01:35:34 +0000568 bool MadeChange = false;
569 // Note that this intentionally skips the entry block.
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +0000570 for (Function::iterator I = std::next(F.begin()), E = F.end(); I != E;) {
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +0000571 BasicBlock *BB = &*I++;
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000572 BasicBlock *DestBB = findDestBlockOfMergeableEmptyBlock(BB);
573 if (!DestBB ||
574 !isMergingEmptyBlockProfitable(BB, DestBB, Preheaders.count(BB)))
Chris Lattnerc3748562007-04-02 01:35:34 +0000575 continue;
Eric Christopherc1ea1492008-09-24 05:32:41 +0000576
Sanjay Patelfc580a62015-09-21 23:03:16 +0000577 eliminateMostlyEmptyBlock(BB);
Chris Lattnerc3748562007-04-02 01:35:34 +0000578 MadeChange = true;
579 }
580 return MadeChange;
581}
582
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000583bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB,
584 BasicBlock *DestBB,
585 bool isPreheader) {
586 // Do not delete loop preheaders if doing so would create a critical edge.
587 // Loop preheaders can be good locations to spill registers. If the
588 // preheader is deleted and we create a critical edge, registers may be
589 // spilled in the loop body instead.
590 if (!DisablePreheaderProtect && isPreheader &&
591 !(BB->getSinglePredecessor() &&
592 BB->getSinglePredecessor()->getSingleSuccessor()))
593 return false;
594
595 // Try to skip merging if the unique predecessor of BB is terminated by a
596 // switch or indirect branch instruction, and BB is used as an incoming block
597 // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to
598 // add COPY instructions in the predecessor of BB instead of BB (if it is not
599 // merged). Note that the critical edge created by merging such blocks wont be
600 // split in MachineSink because the jump table is not analyzable. By keeping
601 // such empty block (BB), ISel will place COPY instructions in BB, not in the
602 // predecessor of BB.
603 BasicBlock *Pred = BB->getUniquePredecessor();
604 if (!Pred ||
605 !(isa<SwitchInst>(Pred->getTerminator()) ||
606 isa<IndirectBrInst>(Pred->getTerminator())))
607 return true;
608
609 if (BB->getTerminator() != BB->getFirstNonPHI())
610 return true;
611
612 // We use a simple cost heuristic which determine skipping merging is
613 // profitable if the cost of skipping merging is less than the cost of
614 // merging : Cost(skipping merging) < Cost(merging BB), where the
615 // Cost(skipping merging) is Freq(BB) * (Cost(Copy) + Cost(Branch)), and
616 // the Cost(merging BB) is Freq(Pred) * Cost(Copy).
617 // Assuming Cost(Copy) == Cost(Branch), we could simplify it to :
618 // Freq(Pred) / Freq(BB) > 2.
619 // Note that if there are multiple empty blocks sharing the same incoming
620 // value for the PHIs in the DestBB, we consider them together. In such
621 // case, Cost(merging BB) will be the sum of their frequencies.
622
623 if (!isa<PHINode>(DestBB->begin()))
624 return true;
625
626 SmallPtrSet<BasicBlock *, 16> SameIncomingValueBBs;
627
628 // Find all other incoming blocks from which incoming values of all PHIs in
629 // DestBB are the same as the ones from BB.
630 for (pred_iterator PI = pred_begin(DestBB), E = pred_end(DestBB); PI != E;
631 ++PI) {
632 BasicBlock *DestBBPred = *PI;
633 if (DestBBPred == BB)
634 continue;
635
Balaram Makam32bcb5d2017-10-27 00:35:18 +0000636 bool HasAllSameValue = true;
637 BasicBlock::const_iterator DestBBI = DestBB->begin();
638 while (const PHINode *DestPN = dyn_cast<PHINode>(DestBBI++)) {
639 if (DestPN->getIncomingValueForBlock(BB) !=
640 DestPN->getIncomingValueForBlock(DestBBPred)) {
641 HasAllSameValue = false;
642 break;
643 }
644 }
645 if (HasAllSameValue)
Jun Bum Lim90b6b502016-12-16 20:38:39 +0000646 SameIncomingValueBBs.insert(DestBBPred);
647 }
648
649 // See if all BB's incoming values are same as the value from Pred. In this
650 // case, no reason to skip merging because COPYs are expected to be place in
651 // Pred already.
652 if (SameIncomingValueBBs.count(Pred))
653 return true;
654
655 if (!BFI) {
656 Function &F = *BB->getParent();
657 LoopInfo LI{DominatorTree(F)};
658 BPI.reset(new BranchProbabilityInfo(F, LI));
659 BFI.reset(new BlockFrequencyInfo(F, *BPI, LI));
660 }
661
662 BlockFrequency PredFreq = BFI->getBlockFreq(Pred);
663 BlockFrequency BBFreq = BFI->getBlockFreq(BB);
664
665 for (auto SameValueBB : SameIncomingValueBBs)
666 if (SameValueBB->getUniquePredecessor() == Pred &&
667 DestBB == findDestBlockOfMergeableEmptyBlock(SameValueBB))
668 BBFreq += BFI->getBlockFreq(SameValueBB);
669
670 return PredFreq.getFrequency() <=
671 BBFreq.getFrequency() * FreqRatioToSkipMerge;
672}
673
Sanjay Patel4ac6b112015-09-21 22:47:23 +0000674/// Return true if we can merge BB into DestBB if there is a single
675/// unconditional branch between them, and BB contains no other non-phi
Chris Lattnerc3748562007-04-02 01:35:34 +0000676/// instructions.
Sanjay Patelfc580a62015-09-21 23:03:16 +0000677bool CodeGenPrepare::canMergeBlocks(const BasicBlock *BB,
Chris Lattnerc3748562007-04-02 01:35:34 +0000678 const BasicBlock *DestBB) const {
679 // We only want to eliminate blocks whose phi nodes are used by phi nodes in
680 // the successor. If there are more complex condition (e.g. preheaders),
681 // don't mess around with them.
682 BasicBlock::const_iterator BBI = BB->begin();
683 while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
Chandler Carruthcdf47882014-03-09 03:16:01 +0000684 for (const User *U : PN->users()) {
685 const Instruction *UI = cast<Instruction>(U);
686 if (UI->getParent() != DestBB || !isa<PHINode>(UI))
Chris Lattnerc3748562007-04-02 01:35:34 +0000687 return false;
Eric Christopherc1ea1492008-09-24 05:32:41 +0000688 // If User is inside DestBB block and it is a PHINode then check
689 // incoming value. If incoming value is not from BB then this is
Devang Pateld3208522007-04-25 00:37:04 +0000690 // a complex condition (e.g. preheaders) we want to avoid here.
Chandler Carruthcdf47882014-03-09 03:16:01 +0000691 if (UI->getParent() == DestBB) {
692 if (const PHINode *UPN = dyn_cast<PHINode>(UI))
Devang Pateld3208522007-04-25 00:37:04 +0000693 for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) {
694 Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I));
695 if (Insn && Insn->getParent() == BB &&
696 Insn->getParent() != UPN->getIncomingBlock(I))
697 return false;
698 }
699 }
Chris Lattnerc3748562007-04-02 01:35:34 +0000700 }
701 }
Eric Christopherc1ea1492008-09-24 05:32:41 +0000702
Chris Lattnerc3748562007-04-02 01:35:34 +0000703 // If BB and DestBB contain any common predecessors, then the phi nodes in BB
704 // and DestBB may have conflicting incoming values for the block. If so, we
705 // can't merge the block.
706 const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin());
707 if (!DestBBPN) return true; // no conflict.
Eric Christopherc1ea1492008-09-24 05:32:41 +0000708
Chris Lattnerc3748562007-04-02 01:35:34 +0000709 // Collect the preds of BB.
Chris Lattner8201a9b2007-11-06 22:07:40 +0000710 SmallPtrSet<const BasicBlock*, 16> BBPreds;
Chris Lattnerc3748562007-04-02 01:35:34 +0000711 if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
712 // It is faster to get preds from a PHI than with pred_iterator.
713 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
714 BBPreds.insert(BBPN->getIncomingBlock(i));
715 } else {
716 BBPreds.insert(pred_begin(BB), pred_end(BB));
717 }
Eric Christopherc1ea1492008-09-24 05:32:41 +0000718
Chris Lattnerc3748562007-04-02 01:35:34 +0000719 // Walk the preds of DestBB.
720 for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) {
721 BasicBlock *Pred = DestBBPN->getIncomingBlock(i);
722 if (BBPreds.count(Pred)) { // Common predecessor?
723 BBI = DestBB->begin();
724 while (const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
725 const Value *V1 = PN->getIncomingValueForBlock(Pred);
726 const Value *V2 = PN->getIncomingValueForBlock(BB);
Eric Christopherc1ea1492008-09-24 05:32:41 +0000727
Chris Lattnerc3748562007-04-02 01:35:34 +0000728 // If V2 is a phi node in BB, look up what the mapped value will be.
729 if (const PHINode *V2PN = dyn_cast<PHINode>(V2))
730 if (V2PN->getParent() == BB)
731 V2 = V2PN->getIncomingValueForBlock(Pred);
Eric Christopherc1ea1492008-09-24 05:32:41 +0000732
Chris Lattnerc3748562007-04-02 01:35:34 +0000733 // If there is a conflict, bail out.
734 if (V1 != V2) return false;
735 }
736 }
737 }
738
739 return true;
740}
741
Sanjay Patel4ac6b112015-09-21 22:47:23 +0000742/// Eliminate a basic block that has only phi's and an unconditional branch in
743/// it.
Sanjay Patelfc580a62015-09-21 23:03:16 +0000744void CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
Chris Lattnerc3748562007-04-02 01:35:34 +0000745 BranchInst *BI = cast<BranchInst>(BB->getTerminator());
746 BasicBlock *DestBB = BI->getSuccessor(0);
Eric Christopherc1ea1492008-09-24 05:32:41 +0000747
David Greene74e2d492010-01-05 01:27:11 +0000748 DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" << *BB << *DestBB);
Eric Christopherc1ea1492008-09-24 05:32:41 +0000749
Chris Lattnerc3748562007-04-02 01:35:34 +0000750 // If the destination block has a single pred, then this is a trivial edge,
751 // just collapse it.
Chris Lattner4059f432008-11-27 19:29:14 +0000752 if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) {
Chris Lattner8a172da2008-11-28 19:54:49 +0000753 if (SinglePred != DestBB) {
754 // Remember if SinglePred was the entry block of the function. If so, we
755 // will need to move BB back to the entry position.
756 bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock();
Balaram Makam32bcb5d2017-10-27 00:35:18 +0000757 MergeBasicBlockIntoOnlyPred(DestBB, nullptr);
Chris Lattner4059f432008-11-27 19:29:14 +0000758
Chris Lattner8a172da2008-11-28 19:54:49 +0000759 if (isEntry && BB != &BB->getParent()->getEntryBlock())
760 BB->moveBefore(&BB->getParent()->getEntryBlock());
Nadav Rotem465834c2012-07-24 10:51:42 +0000761
David Greene74e2d492010-01-05 01:27:11 +0000762 DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");
Chris Lattner8a172da2008-11-28 19:54:49 +0000763 return;
764 }
Chris Lattnerc3748562007-04-02 01:35:34 +0000765 }
Eric Christopherc1ea1492008-09-24 05:32:41 +0000766
Chris Lattnerc3748562007-04-02 01:35:34 +0000767 // Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB
768 // to handle the new incoming edges it is about to have.
769 PHINode *PN;
770 for (BasicBlock::iterator BBI = DestBB->begin();
771 (PN = dyn_cast<PHINode>(BBI)); ++BBI) {
772 // Remove the incoming value for BB, and remember it.
773 Value *InVal = PN->removeIncomingValue(BB, false);
Eric Christopherc1ea1492008-09-24 05:32:41 +0000774
Chris Lattnerc3748562007-04-02 01:35:34 +0000775 // Two options: either the InVal is a phi node defined in BB or it is some
776 // value that dominates BB.
777 PHINode *InValPhi = dyn_cast<PHINode>(InVal);
778 if (InValPhi && InValPhi->getParent() == BB) {
779 // Add all of the input values of the input PHI as inputs of this phi.
780 for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i)
781 PN->addIncoming(InValPhi->getIncomingValue(i),
782 InValPhi->getIncomingBlock(i));
783 } else {
784 // Otherwise, add one instance of the dominating value for each edge that
785 // we will be adding.
786 if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {
787 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
788 PN->addIncoming(InVal, BBPN->getIncomingBlock(i));
789 } else {
Duncan P. N. Exon Smith6c990152014-07-21 17:06:51 +0000790 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
791 PN->addIncoming(InVal, *PI);
Chris Lattnerc3748562007-04-02 01:35:34 +0000792 }
793 }
794 }
Eric Christopherc1ea1492008-09-24 05:32:41 +0000795
Chris Lattnerc3748562007-04-02 01:35:34 +0000796 // The PHIs are now updated, change everything that refers to BB to use
797 // DestBB and remove BB.
798 BB->replaceAllUsesWith(DestBB);
799 BB->eraseFromParent();
Cameron Zwarichced753f2011-01-05 17:27:27 +0000800 ++NumBlocksElim;
Eric Christopherc1ea1492008-09-24 05:32:41 +0000801
David Greene74e2d492010-01-05 01:27:11 +0000802 DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");
Chris Lattnerc3748562007-04-02 01:35:34 +0000803}
804
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000805// Computes a map of base pointer relocation instructions to corresponding
806// derived pointer relocation instructions given a vector of all relocate calls
807static void computeBaseDerivedRelocateMap(
Manuel Jacob83eefa62016-01-05 04:03:00 +0000808 const SmallVectorImpl<GCRelocateInst *> &AllRelocateCalls,
809 DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>>
810 &RelocateInstMap) {
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000811 // Collect information in two maps: one primarily for locating the base object
812 // while filling the second map; the second map is the final structure holding
813 // a mapping between Base and corresponding Derived relocate calls
Manuel Jacob83eefa62016-01-05 04:03:00 +0000814 DenseMap<std::pair<unsigned, unsigned>, GCRelocateInst *> RelocateIdxMap;
815 for (auto *ThisRelocate : AllRelocateCalls) {
816 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),
817 ThisRelocate->getDerivedPtrIndex());
818 RelocateIdxMap.insert(std::make_pair(K, ThisRelocate));
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000819 }
820 for (auto &Item : RelocateIdxMap) {
821 std::pair<unsigned, unsigned> Key = Item.first;
822 if (Key.first == Key.second)
823 // Base relocation: nothing to insert
824 continue;
825
Manuel Jacob83eefa62016-01-05 04:03:00 +0000826 GCRelocateInst *I = Item.second;
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000827 auto BaseKey = std::make_pair(Key.first, Key.first);
Sanjoy Dasb8186762015-02-27 02:24:16 +0000828
829 // We're iterating over RelocateIdxMap so we cannot modify it.
830 auto MaybeBase = RelocateIdxMap.find(BaseKey);
831 if (MaybeBase == RelocateIdxMap.end())
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000832 // TODO: We might want to insert a new base object relocate and gep off
833 // that, if there are enough derived object relocates.
834 continue;
Sanjoy Dasb8186762015-02-27 02:24:16 +0000835
836 RelocateInstMap[MaybeBase->second].push_back(I);
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000837 }
838}
839
840// Accepts a GEP and extracts the operands into a vector provided they're all
841// small integer constants
842static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP,
843 SmallVectorImpl<Value *> &OffsetV) {
844 for (unsigned i = 1; i < GEP->getNumOperands(); i++) {
845 // Only accept small constant integer operands
846 auto Op = dyn_cast<ConstantInt>(GEP->getOperand(i));
847 if (!Op || Op->getZExtValue() > 20)
848 return false;
849 }
850
851 for (unsigned i = 1; i < GEP->getNumOperands(); i++)
852 OffsetV.push_back(GEP->getOperand(i));
853 return true;
854}
855
856// Takes a RelocatedBase (base pointer relocation instruction) and Targets to
857// replace, computes a replacement, and affects it.
858static bool
Manuel Jacob83eefa62016-01-05 04:03:00 +0000859simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase,
860 const SmallVectorImpl<GCRelocateInst *> &Targets) {
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000861 bool MadeChange = false;
Serguei Katkov9e5604d2017-08-17 05:48:30 +0000862 // We must ensure the relocation of derived pointer is defined after
863 // relocation of base pointer. If we find a relocation corresponding to base
864 // defined earlier than relocation of base then we move relocation of base
865 // right before found relocation. We consider only relocation in the same
866 // basic block as relocation of base. Relocations from other basic block will
867 // be skipped by optimization and we do not care about them.
868 for (auto R = RelocatedBase->getParent()->getFirstInsertionPt();
869 &*R != RelocatedBase; ++R)
870 if (auto RI = dyn_cast<GCRelocateInst>(R))
871 if (RI->getStatepoint() == RelocatedBase->getStatepoint())
872 if (RI->getBasePtrIndex() == RelocatedBase->getBasePtrIndex()) {
873 RelocatedBase->moveBefore(RI);
874 break;
875 }
876
Manuel Jacob83eefa62016-01-05 04:03:00 +0000877 for (GCRelocateInst *ToReplace : Targets) {
878 assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() &&
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000879 "Not relocating a derived object of the original base object");
Manuel Jacob83eefa62016-01-05 04:03:00 +0000880 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000881 // A duplicate relocate call. TODO: coalesce duplicates.
882 continue;
883 }
884
Igor Laevskyf637b4a2015-11-03 18:37:40 +0000885 if (RelocatedBase->getParent() != ToReplace->getParent()) {
886 // Base and derived relocates are in different basic blocks.
887 // In this case transform is only valid when base dominates derived
888 // relocate. However it would be too expensive to check dominance
889 // for each such relocate, so we skip the whole transformation.
890 continue;
891 }
892
Manuel Jacob83eefa62016-01-05 04:03:00 +0000893 Value *Base = ToReplace->getBasePtr();
894 auto Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr());
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000895 if (!Derived || Derived->getPointerOperand() != Base)
896 continue;
897
898 SmallVector<Value *, 2> OffsetV;
899 if (!getGEPSmallConstantIntOffsetV(Derived, OffsetV))
900 continue;
901
902 // Create a Builder and replace the target callsite with a gep
Sanjay Patel545a4562016-01-20 18:59:16 +0000903 assert(RelocatedBase->getNextNode() &&
904 "Should always have one since it's not a terminator");
Sanjoy Das3d705e32015-05-11 23:47:30 +0000905
906 // Insert after RelocatedBase
907 IRBuilder<> Builder(RelocatedBase->getNextNode());
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000908 Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
Sanjoy Das89c54912015-05-11 18:49:34 +0000909
910 // If gc_relocate does not match the actual type, cast it to the right type.
911 // In theory, there must be a bitcast after gc_relocate if the type does not
912 // match, and we should reuse it to get the derived pointer. But it could be
913 // cases like this:
914 // bb1:
915 // ...
916 // %g1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
917 // br label %merge
918 //
919 // bb2:
920 // ...
921 // %g2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
922 // br label %merge
923 //
924 // merge:
925 // %p1 = phi i8 addrspace(1)* [ %g1, %bb1 ], [ %g2, %bb2 ]
926 // %cast = bitcast i8 addrspace(1)* %p1 in to i32 addrspace(1)*
927 //
928 // In this case, we can not find the bitcast any more. So we insert a new bitcast
929 // no matter there is already one or not. In this way, we can handle all cases, and
930 // the extra bitcast should be optimized away in later passes.
Manuel Jacob5b90b142015-12-19 18:38:42 +0000931 Value *ActualRelocatedBase = RelocatedBase;
Sanjoy Das89c54912015-05-11 18:49:34 +0000932 if (RelocatedBase->getType() != Base->getType()) {
933 ActualRelocatedBase =
Manuel Jacob5b90b142015-12-19 18:38:42 +0000934 Builder.CreateBitCast(RelocatedBase, Base->getType());
Sanjoy Das89c54912015-05-11 18:49:34 +0000935 }
David Blaikie68d535c2015-03-24 22:38:16 +0000936 Value *Replacement = Builder.CreateGEP(
Sanjoy Das89c54912015-05-11 18:49:34 +0000937 Derived->getSourceElementType(), ActualRelocatedBase, makeArrayRef(OffsetV));
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000938 Replacement->takeName(ToReplace);
Sanjoy Das89c54912015-05-11 18:49:34 +0000939 // If the newly generated derived pointer's type does not match the original derived
940 // pointer's type, cast the new derived pointer to match it. Same reasoning as above.
Manuel Jacob5b90b142015-12-19 18:38:42 +0000941 Value *ActualReplacement = Replacement;
942 if (Replacement->getType() != ToReplace->getType()) {
Sanjoy Das89c54912015-05-11 18:49:34 +0000943 ActualReplacement =
Manuel Jacob5b90b142015-12-19 18:38:42 +0000944 Builder.CreateBitCast(Replacement, ToReplace->getType());
Sanjoy Das89c54912015-05-11 18:49:34 +0000945 }
946 ToReplace->replaceAllUsesWith(ActualReplacement);
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000947 ToReplace->eraseFromParent();
948
949 MadeChange = true;
950 }
951 return MadeChange;
952}
953
954// Turns this:
955//
956// %base = ...
957// %ptr = gep %base + 15
958// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
959// %base' = relocate(%tok, i32 4, i32 4)
960// %ptr' = relocate(%tok, i32 4, i32 5)
961// %val = load %ptr'
962//
963// into this:
964//
965// %base = ...
966// %ptr = gep %base + 15
967// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
968// %base' = gc.relocate(%tok, i32 4, i32 4)
969// %ptr' = gep %base' + 15
970// %val = load %ptr'
971bool CodeGenPrepare::simplifyOffsetableRelocate(Instruction &I) {
972 bool MadeChange = false;
Manuel Jacob83eefa62016-01-05 04:03:00 +0000973 SmallVector<GCRelocateInst *, 2> AllRelocateCalls;
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000974
975 for (auto *U : I.users())
Manuel Jacob83eefa62016-01-05 04:03:00 +0000976 if (GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U))
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000977 // Collect all the relocate calls associated with a statepoint
Manuel Jacob83eefa62016-01-05 04:03:00 +0000978 AllRelocateCalls.push_back(Relocate);
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000979
980 // We need atleast one base pointer relocation + one derived pointer
981 // relocation to mangle
982 if (AllRelocateCalls.size() < 2)
983 return false;
984
985 // RelocateInstMap is a mapping from the base relocate instruction to the
986 // corresponding derived relocate instructions
Manuel Jacob83eefa62016-01-05 04:03:00 +0000987 DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>> RelocateInstMap;
Ramkumar Ramachandradba73292015-01-14 23:27:07 +0000988 computeBaseDerivedRelocateMap(AllRelocateCalls, RelocateInstMap);
989 if (RelocateInstMap.empty())
990 return false;
991
992 for (auto &Item : RelocateInstMap)
993 // Item.first is the RelocatedBase to offset against
994 // Item.second is the vector of Targets to replace
995 MadeChange = simplifyRelocatesOffABase(Item.first, Item.second);
996 return MadeChange;
997}
998
Manuel Jacoba7c48f92014-03-13 13:36:25 +0000999/// SinkCast - Sink the specified cast instruction into its user blocks
1000static bool SinkCast(CastInst *CI) {
Chris Lattnerf2836d12007-03-31 04:06:36 +00001001 BasicBlock *DefBB = CI->getParent();
Eric Christopherc1ea1492008-09-24 05:32:41 +00001002
Chris Lattnerf2836d12007-03-31 04:06:36 +00001003 /// InsertedCasts - Only insert a cast in each block once.
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001004 DenseMap<BasicBlock*, CastInst*> InsertedCasts;
Eric Christopherc1ea1492008-09-24 05:32:41 +00001005
Chris Lattnerf2836d12007-03-31 04:06:36 +00001006 bool MadeChange = false;
Chandler Carruthcdf47882014-03-09 03:16:01 +00001007 for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();
Chris Lattnerf2836d12007-03-31 04:06:36 +00001008 UI != E; ) {
1009 Use &TheUse = UI.getUse();
1010 Instruction *User = cast<Instruction>(*UI);
Eric Christopherc1ea1492008-09-24 05:32:41 +00001011
Chris Lattnerf2836d12007-03-31 04:06:36 +00001012 // Figure out which BB this cast is used in. For PHI's this is the
1013 // appropriate predecessor block.
1014 BasicBlock *UserBB = User->getParent();
1015 if (PHINode *PN = dyn_cast<PHINode>(User)) {
Chandler Carruthcdf47882014-03-09 03:16:01 +00001016 UserBB = PN->getIncomingBlock(TheUse);
Chris Lattnerf2836d12007-03-31 04:06:36 +00001017 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00001018
Chris Lattnerf2836d12007-03-31 04:06:36 +00001019 // Preincrement use iterator so we don't invalidate it.
1020 ++UI;
Eric Christopherc1ea1492008-09-24 05:32:41 +00001021
David Majnemer0c80e2e2016-04-27 19:36:38 +00001022 // The first insertion point of a block containing an EH pad is after the
1023 // pad. If the pad is the user, we cannot sink the cast past the pad.
1024 if (User->isEHPad())
1025 continue;
1026
Andrew Kaylord0430e82015-11-23 19:16:15 +00001027 // If the block selected to receive the cast is an EH pad that does not
1028 // allow non-PHI instructions before the terminator, we can't sink the
1029 // cast.
1030 if (UserBB->getTerminator()->isEHPad())
1031 continue;
1032
Chris Lattnerf2836d12007-03-31 04:06:36 +00001033 // If this user is in the same block as the cast, don't change the cast.
1034 if (UserBB == DefBB) continue;
Eric Christopherc1ea1492008-09-24 05:32:41 +00001035
Chris Lattnerf2836d12007-03-31 04:06:36 +00001036 // If we have already inserted a cast into this block, use it.
1037 CastInst *&InsertedCast = InsertedCasts[UserBB];
1038
1039 if (!InsertedCast) {
Bill Wendling8ddfc092011-08-16 20:45:24 +00001040 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001041 assert(InsertPt != UserBB->end());
1042 InsertedCast = CastInst::Create(CI->getOpcode(), CI->getOperand(0),
1043 CI->getType(), "", &*InsertPt);
Chris Lattnerf2836d12007-03-31 04:06:36 +00001044 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00001045
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001046 // Replace a use of the cast with a use of the new cast.
Chris Lattnerf2836d12007-03-31 04:06:36 +00001047 TheUse = InsertedCast;
Benjamin Kramerb4bf14c2015-04-10 22:25:36 +00001048 MadeChange = true;
Cameron Zwarichced753f2011-01-05 17:27:27 +00001049 ++NumCastUses;
Chris Lattnerf2836d12007-03-31 04:06:36 +00001050 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00001051
Chris Lattnerf2836d12007-03-31 04:06:36 +00001052 // If we removed all uses, nuke the cast.
Duncan Sandsafa84da42008-01-20 16:51:46 +00001053 if (CI->use_empty()) {
Adrian Prantl261ac8b2017-11-03 21:55:03 +00001054 salvageDebugInfo(*CI);
Chris Lattnerf2836d12007-03-31 04:06:36 +00001055 CI->eraseFromParent();
Duncan Sandsafa84da42008-01-20 16:51:46 +00001056 MadeChange = true;
1057 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00001058
Chris Lattnerf2836d12007-03-31 04:06:36 +00001059 return MadeChange;
1060}
1061
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001062/// If the specified cast instruction is a noop copy (e.g. it's casting from
1063/// one pointer type to another, i32->i8 on PPC), sink it into user blocks to
1064/// reduce the number of virtual registers that must be created and coalesced.
Manuel Jacoba7c48f92014-03-13 13:36:25 +00001065///
1066/// Return true if any changes are made.
Mehdi Amini44ede332015-07-09 02:09:04 +00001067static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI,
1068 const DataLayout &DL) {
Justin Lebar3e50a5b2016-11-21 22:49:15 +00001069 // Sink only "cheap" (or nop) address-space casts. This is a weaker condition
1070 // than sinking only nop casts, but is helpful on some platforms.
1071 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(CI)) {
1072 if (!TLI.isCheapAddrSpaceCast(ASC->getSrcAddressSpace(),
1073 ASC->getDestAddressSpace()))
1074 return false;
1075 }
1076
Manuel Jacoba7c48f92014-03-13 13:36:25 +00001077 // If this is a noop copy,
Mehdi Amini44ede332015-07-09 02:09:04 +00001078 EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType());
1079 EVT DstVT = TLI.getValueType(DL, CI->getType());
Manuel Jacoba7c48f92014-03-13 13:36:25 +00001080
1081 // This is an fp<->int conversion?
1082 if (SrcVT.isInteger() != DstVT.isInteger())
1083 return false;
1084
1085 // If this is an extension, it will be a zero or sign extension, which
1086 // isn't a noop.
1087 if (SrcVT.bitsLT(DstVT)) return false;
1088
1089 // If these values will be promoted, find out what they will be promoted
1090 // to. This helps us consider truncates on PPC as noop copies when they
1091 // are.
1092 if (TLI.getTypeAction(CI->getContext(), SrcVT) ==
1093 TargetLowering::TypePromoteInteger)
1094 SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT);
1095 if (TLI.getTypeAction(CI->getContext(), DstVT) ==
1096 TargetLowering::TypePromoteInteger)
1097 DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT);
1098
1099 // If, after promotion, these are the same types, this is a noop copy.
1100 if (SrcVT != DstVT)
1101 return false;
1102
1103 return SinkCast(CI);
1104}
1105
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001106/// Try to combine CI into a call to the llvm.uadd.with.overflow intrinsic if
1107/// possible.
Sanjoy Dasb6c59142015-04-10 21:07:09 +00001108///
1109/// Return true if any changes were made.
1110static bool CombineUAddWithOverflow(CmpInst *CI) {
1111 Value *A, *B;
1112 Instruction *AddI;
1113 if (!match(CI,
1114 m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI))))
1115 return false;
1116
1117 Type *Ty = AddI->getType();
1118 if (!isa<IntegerType>(Ty))
1119 return false;
1120
1121 // We don't want to move around uses of condition values this late, so we we
1122 // check if it is legal to create the call to the intrinsic in the basic
1123 // block containing the icmp:
1124
1125 if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse())
1126 return false;
1127
1128#ifndef NDEBUG
1129 // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption
1130 // for now:
1131 if (AddI->hasOneUse())
1132 assert(*AddI->user_begin() == CI && "expected!");
1133#endif
1134
Sanjay Patelaf674fb2015-12-14 17:24:23 +00001135 Module *M = CI->getModule();
Sanjoy Dasb6c59142015-04-10 21:07:09 +00001136 Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty);
1137
1138 auto *InsertPt = AddI->hasOneUse() ? CI : AddI;
1139
1140 auto *UAddWithOverflow =
1141 CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt);
1142 auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt);
1143 auto *Overflow =
1144 ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt);
1145
1146 CI->replaceAllUsesWith(Overflow);
1147 AddI->replaceAllUsesWith(UAdd);
1148 CI->eraseFromParent();
1149 AddI->eraseFromParent();
1150 return true;
1151}
1152
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001153/// Sink the given CmpInst into user blocks to reduce the number of virtual
1154/// registers that must be created and coalesced. This is a clear win except on
1155/// targets with multiple condition code registers (PowerPC), where it might
1156/// lose; some adjustment may be wanted there.
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001157///
1158/// Return true if any changes are made.
Peter Zotov8efe38a2016-04-03 19:32:13 +00001159static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001160 BasicBlock *DefBB = CI->getParent();
Eric Christopherc1ea1492008-09-24 05:32:41 +00001161
Peter Zotov0b6d7bc2016-04-03 16:36:17 +00001162 // Avoid sinking soft-FP comparisons, since this can move them into a loop.
Peter Zotov8efe38a2016-04-03 19:32:13 +00001163 if (TLI && TLI->useSoftFloat() && isa<FCmpInst>(CI))
Peter Zotov0b6d7bc2016-04-03 16:36:17 +00001164 return false;
1165
1166 // Only insert a cmp in each block once.
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001167 DenseMap<BasicBlock*, CmpInst*> InsertedCmps;
Eric Christopherc1ea1492008-09-24 05:32:41 +00001168
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001169 bool MadeChange = false;
Chandler Carruthcdf47882014-03-09 03:16:01 +00001170 for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001171 UI != E; ) {
1172 Use &TheUse = UI.getUse();
1173 Instruction *User = cast<Instruction>(*UI);
Eric Christopherc1ea1492008-09-24 05:32:41 +00001174
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001175 // Preincrement use iterator so we don't invalidate it.
1176 ++UI;
Eric Christopherc1ea1492008-09-24 05:32:41 +00001177
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001178 // Don't bother for PHI nodes.
1179 if (isa<PHINode>(User))
1180 continue;
1181
1182 // Figure out which BB this cmp is used in.
1183 BasicBlock *UserBB = User->getParent();
Eric Christopherc1ea1492008-09-24 05:32:41 +00001184
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001185 // If this user is in the same block as the cmp, don't change the cmp.
1186 if (UserBB == DefBB) continue;
Eric Christopherc1ea1492008-09-24 05:32:41 +00001187
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001188 // If we have already inserted a cmp into this block, use it.
1189 CmpInst *&InsertedCmp = InsertedCmps[UserBB];
1190
1191 if (!InsertedCmp) {
Bill Wendling8ddfc092011-08-16 20:45:24 +00001192 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001193 assert(InsertPt != UserBB->end());
Eric Christopherc1ea1492008-09-24 05:32:41 +00001194 InsertedCmp =
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001195 CmpInst::Create(CI->getOpcode(), CI->getPredicate(),
1196 CI->getOperand(0), CI->getOperand(1), "", &*InsertPt);
Wolfgang Piebe51bede2016-10-06 21:43:45 +00001197 // Propagate the debug info.
1198 InsertedCmp->setDebugLoc(CI->getDebugLoc());
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001199 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00001200
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001201 // Replace a use of the cmp with a use of the new cmp.
1202 TheUse = InsertedCmp;
Benjamin Kramerb4bf14c2015-04-10 22:25:36 +00001203 MadeChange = true;
Cameron Zwarichced753f2011-01-05 17:27:27 +00001204 ++NumCmpUses;
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001205 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00001206
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001207 // If we removed all uses, nuke the cmp.
Benjamin Kramerb4bf14c2015-04-10 22:25:36 +00001208 if (CI->use_empty()) {
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001209 CI->eraseFromParent();
Benjamin Kramerb4bf14c2015-04-10 22:25:36 +00001210 MadeChange = true;
1211 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00001212
Dale Johannesenedfec0b2007-06-12 16:50:17 +00001213 return MadeChange;
1214}
1215
Peter Zotovf87e5502016-04-03 17:11:53 +00001216static bool OptimizeCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
Peter Zotov8efe38a2016-04-03 19:32:13 +00001217 if (SinkCmpExpression(CI, TLI))
Sanjoy Dasb6c59142015-04-10 21:07:09 +00001218 return true;
1219
1220 if (CombineUAddWithOverflow(CI))
1221 return true;
1222
1223 return false;
1224}
1225
Geoff Berry5d534b62017-02-21 18:53:14 +00001226/// Duplicate and sink the given 'and' instruction into user blocks where it is
1227/// used in a compare to allow isel to generate better code for targets where
1228/// this operation can be combined.
1229///
1230/// Return true if any changes are made.
1231static bool sinkAndCmp0Expression(Instruction *AndI,
1232 const TargetLowering &TLI,
1233 SetOfInstrs &InsertedInsts) {
1234 // Double-check that we're not trying to optimize an instruction that was
1235 // already optimized by some other part of this pass.
1236 assert(!InsertedInsts.count(AndI) &&
1237 "Attempting to optimize already optimized and instruction");
1238 (void) InsertedInsts;
1239
1240 // Nothing to do for single use in same basic block.
1241 if (AndI->hasOneUse() &&
1242 AndI->getParent() == cast<Instruction>(*AndI->user_begin())->getParent())
1243 return false;
1244
1245 // Try to avoid cases where sinking/duplicating is likely to increase register
1246 // pressure.
1247 if (!isa<ConstantInt>(AndI->getOperand(0)) &&
1248 !isa<ConstantInt>(AndI->getOperand(1)) &&
1249 AndI->getOperand(0)->hasOneUse() && AndI->getOperand(1)->hasOneUse())
1250 return false;
1251
1252 for (auto *U : AndI->users()) {
1253 Instruction *User = cast<Instruction>(U);
1254
1255 // Only sink for and mask feeding icmp with 0.
1256 if (!isa<ICmpInst>(User))
1257 return false;
1258
1259 auto *CmpC = dyn_cast<ConstantInt>(User->getOperand(1));
1260 if (!CmpC || !CmpC->isZero())
1261 return false;
1262 }
1263
1264 if (!TLI.isMaskAndCmp0FoldingBeneficial(*AndI))
1265 return false;
1266
1267 DEBUG(dbgs() << "found 'and' feeding only icmp 0;\n");
1268 DEBUG(AndI->getParent()->dump());
1269
1270 // Push the 'and' into the same block as the icmp 0. There should only be
1271 // one (icmp (and, 0)) in each block, since CSE/GVN should have removed any
1272 // others, so we don't need to keep track of which BBs we insert into.
1273 for (Value::user_iterator UI = AndI->user_begin(), E = AndI->user_end();
1274 UI != E; ) {
1275 Use &TheUse = UI.getUse();
1276 Instruction *User = cast<Instruction>(*UI);
1277
1278 // Preincrement use iterator so we don't invalidate it.
1279 ++UI;
1280
1281 DEBUG(dbgs() << "sinking 'and' use: " << *User << "\n");
1282
1283 // Keep the 'and' in the same place if the use is already in the same block.
1284 Instruction *InsertPt =
1285 User->getParent() == AndI->getParent() ? AndI : User;
1286 Instruction *InsertedAnd =
1287 BinaryOperator::Create(Instruction::And, AndI->getOperand(0),
1288 AndI->getOperand(1), "", InsertPt);
1289 // Propagate the debug info.
1290 InsertedAnd->setDebugLoc(AndI->getDebugLoc());
1291
1292 // Replace a use of the 'and' with a use of the new 'and'.
1293 TheUse = InsertedAnd;
1294 ++NumAndUses;
1295 DEBUG(User->getParent()->dump());
1296 }
1297
1298 // We removed all uses, nuke the and.
1299 AndI->eraseFromParent();
1300 return true;
1301}
1302
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001303/// Check if the candidates could be combined with a shift instruction, which
1304/// includes:
Yi Jiangd069f632014-04-21 19:34:27 +00001305/// 1. Truncate instruction
1306/// 2. And instruction and the imm is a mask of the low bits:
1307/// imm & (imm+1) == 0
Benjamin Kramer322053c2014-04-27 14:54:59 +00001308static bool isExtractBitsCandidateUse(Instruction *User) {
Yi Jiangd069f632014-04-21 19:34:27 +00001309 if (!isa<TruncInst>(User)) {
1310 if (User->getOpcode() != Instruction::And ||
1311 !isa<ConstantInt>(User->getOperand(1)))
1312 return false;
1313
Quentin Colombetd4f44692014-04-22 01:20:34 +00001314 const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue();
Yi Jiangd069f632014-04-21 19:34:27 +00001315
Quentin Colombetd4f44692014-04-22 01:20:34 +00001316 if ((Cimm & (Cimm + 1)).getBoolValue())
Yi Jiangd069f632014-04-21 19:34:27 +00001317 return false;
1318 }
1319 return true;
1320}
1321
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001322/// Sink both shift and truncate instruction to the use of truncate's BB.
Benjamin Kramer322053c2014-04-27 14:54:59 +00001323static bool
Yi Jiangd069f632014-04-21 19:34:27 +00001324SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,
1325 DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts,
Mehdi Amini44ede332015-07-09 02:09:04 +00001326 const TargetLowering &TLI, const DataLayout &DL) {
Yi Jiangd069f632014-04-21 19:34:27 +00001327 BasicBlock *UserBB = User->getParent();
1328 DenseMap<BasicBlock *, CastInst *> InsertedTruncs;
1329 TruncInst *TruncI = dyn_cast<TruncInst>(User);
1330 bool MadeChange = false;
1331
1332 for (Value::user_iterator TruncUI = TruncI->user_begin(),
1333 TruncE = TruncI->user_end();
1334 TruncUI != TruncE;) {
1335
1336 Use &TruncTheUse = TruncUI.getUse();
1337 Instruction *TruncUser = cast<Instruction>(*TruncUI);
1338 // Preincrement use iterator so we don't invalidate it.
1339
1340 ++TruncUI;
1341
1342 int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode());
1343 if (!ISDOpcode)
1344 continue;
1345
Tim Northovere2239ff2014-07-29 10:20:22 +00001346 // If the use is actually a legal node, there will not be an
1347 // implicit truncate.
1348 // FIXME: always querying the result type is just an
1349 // approximation; some nodes' legality is determined by the
1350 // operand or other means. There's no good way to find out though.
Ahmed Bougacha0788d492014-11-12 22:16:55 +00001351 if (TLI.isOperationLegalOrCustom(
Mehdi Amini44ede332015-07-09 02:09:04 +00001352 ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true)))
Yi Jiangd069f632014-04-21 19:34:27 +00001353 continue;
1354
1355 // Don't bother for PHI nodes.
1356 if (isa<PHINode>(TruncUser))
1357 continue;
1358
1359 BasicBlock *TruncUserBB = TruncUser->getParent();
1360
1361 if (UserBB == TruncUserBB)
1362 continue;
1363
1364 BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB];
1365 CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];
1366
1367 if (!InsertedShift && !InsertedTrunc) {
1368 BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt();
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001369 assert(InsertPt != TruncUserBB->end());
Yi Jiangd069f632014-04-21 19:34:27 +00001370 // Sink the shift
1371 if (ShiftI->getOpcode() == Instruction::AShr)
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001372 InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
1373 "", &*InsertPt);
Yi Jiangd069f632014-04-21 19:34:27 +00001374 else
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001375 InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
1376 "", &*InsertPt);
Yi Jiangd069f632014-04-21 19:34:27 +00001377
1378 // Sink the trunc
1379 BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt();
1380 TruncInsertPt++;
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001381 assert(TruncInsertPt != TruncUserBB->end());
Yi Jiangd069f632014-04-21 19:34:27 +00001382
1383 InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift,
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001384 TruncI->getType(), "", &*TruncInsertPt);
Yi Jiangd069f632014-04-21 19:34:27 +00001385
1386 MadeChange = true;
1387
1388 TruncTheUse = InsertedTrunc;
1389 }
1390 }
1391 return MadeChange;
1392}
1393
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001394/// Sink the shift *right* instruction into user blocks if the uses could
1395/// potentially be combined with this shift instruction and generate BitExtract
1396/// instruction. It will only be applied if the architecture supports BitExtract
1397/// instruction. Here is an example:
Yi Jiangd069f632014-04-21 19:34:27 +00001398/// BB1:
1399/// %x.extract.shift = lshr i64 %arg1, 32
1400/// BB2:
1401/// %x.extract.trunc = trunc i64 %x.extract.shift to i16
1402/// ==>
1403///
1404/// BB2:
1405/// %x.extract.shift.1 = lshr i64 %arg1, 32
1406/// %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16
1407///
1408/// CodeGen will recoginze the pattern in BB2 and generate BitExtract
1409/// instruction.
1410/// Return true if any changes are made.
1411static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,
Mehdi Amini44ede332015-07-09 02:09:04 +00001412 const TargetLowering &TLI,
1413 const DataLayout &DL) {
Yi Jiangd069f632014-04-21 19:34:27 +00001414 BasicBlock *DefBB = ShiftI->getParent();
1415
1416 /// Only insert instructions in each block once.
1417 DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts;
1418
Mehdi Amini44ede332015-07-09 02:09:04 +00001419 bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType()));
Yi Jiangd069f632014-04-21 19:34:27 +00001420
1421 bool MadeChange = false;
1422 for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end();
1423 UI != E;) {
1424 Use &TheUse = UI.getUse();
1425 Instruction *User = cast<Instruction>(*UI);
1426 // Preincrement use iterator so we don't invalidate it.
1427 ++UI;
1428
1429 // Don't bother for PHI nodes.
1430 if (isa<PHINode>(User))
1431 continue;
1432
1433 if (!isExtractBitsCandidateUse(User))
1434 continue;
1435
1436 BasicBlock *UserBB = User->getParent();
1437
1438 if (UserBB == DefBB) {
1439 // If the shift and truncate instruction are in the same BB. The use of
1440 // the truncate(TruncUse) may still introduce another truncate if not
1441 // legal. In this case, we would like to sink both shift and truncate
1442 // instruction to the BB of TruncUse.
1443 // for example:
1444 // BB1:
1445 // i64 shift.result = lshr i64 opnd, imm
1446 // trunc.result = trunc shift.result to i16
1447 //
1448 // BB2:
1449 // ----> We will have an implicit truncate here if the architecture does
1450 // not have i16 compare.
1451 // cmp i16 trunc.result, opnd2
1452 //
1453 if (isa<TruncInst>(User) && shiftIsLegal
1454 // If the type of the truncate is legal, no trucate will be
1455 // introduced in other basic blocks.
Mehdi Amini44ede332015-07-09 02:09:04 +00001456 &&
1457 (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType()))))
Yi Jiangd069f632014-04-21 19:34:27 +00001458 MadeChange =
Mehdi Amini44ede332015-07-09 02:09:04 +00001459 SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI, DL);
Yi Jiangd069f632014-04-21 19:34:27 +00001460
1461 continue;
1462 }
1463 // If we have already inserted a shift into this block, use it.
1464 BinaryOperator *&InsertedShift = InsertedShifts[UserBB];
1465
1466 if (!InsertedShift) {
1467 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001468 assert(InsertPt != UserBB->end());
Yi Jiangd069f632014-04-21 19:34:27 +00001469
1470 if (ShiftI->getOpcode() == Instruction::AShr)
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001471 InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
1472 "", &*InsertPt);
Yi Jiangd069f632014-04-21 19:34:27 +00001473 else
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00001474 InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
1475 "", &*InsertPt);
Yi Jiangd069f632014-04-21 19:34:27 +00001476
1477 MadeChange = true;
1478 }
1479
1480 // Replace a use of the shift with a use of the new shift.
1481 TheUse = InsertedShift;
1482 }
1483
1484 // If we removed all uses, nuke the shift.
1485 if (ShiftI->use_empty())
1486 ShiftI->eraseFromParent();
1487
1488 return MadeChange;
1489}
1490
Sanjay Patel4699b8a2015-11-19 16:37:10 +00001491/// If counting leading or trailing zeros is an expensive operation and a zero
1492/// input is defined, add a check for zero to avoid calling the intrinsic.
1493///
1494/// We want to transform:
1495/// %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)
1496///
1497/// into:
1498/// entry:
1499/// %cmpz = icmp eq i64 %A, 0
1500/// br i1 %cmpz, label %cond.end, label %cond.false
1501/// cond.false:
1502/// %z = call i64 @llvm.cttz.i64(i64 %A, i1 true)
1503/// br label %cond.end
1504/// cond.end:
1505/// %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
1506///
1507/// If the transform is performed, return true and set ModifiedDT to true.
1508static bool despeculateCountZeros(IntrinsicInst *CountZeros,
1509 const TargetLowering *TLI,
1510 const DataLayout *DL,
1511 bool &ModifiedDT) {
1512 if (!TLI || !DL)
1513 return false;
1514
1515 // If a zero input is undefined, it doesn't make sense to despeculate that.
1516 if (match(CountZeros->getOperand(1), m_One()))
1517 return false;
1518
1519 // If it's cheap to speculate, there's nothing to do.
1520 auto IntrinsicID = CountZeros->getIntrinsicID();
1521 if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz()) ||
1522 (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz()))
1523 return false;
1524
1525 // Only handle legal scalar cases. Anything else requires too much work.
1526 Type *Ty = CountZeros->getType();
1527 unsigned SizeInBits = Ty->getPrimitiveSizeInBits();
Jun Bum Limbe11bdc2016-05-13 18:38:35 +00001528 if (Ty->isVectorTy() || SizeInBits > DL->getLargestLegalIntTypeSizeInBits())
Sanjay Patel4699b8a2015-11-19 16:37:10 +00001529 return false;
1530
1531 // The intrinsic will be sunk behind a compare against zero and branch.
1532 BasicBlock *StartBlock = CountZeros->getParent();
1533 BasicBlock *CallBlock = StartBlock->splitBasicBlock(CountZeros, "cond.false");
1534
1535 // Create another block after the count zero intrinsic. A PHI will be added
1536 // in this block to select the result of the intrinsic or the bit-width
1537 // constant if the input to the intrinsic is zero.
1538 BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(CountZeros));
1539 BasicBlock *EndBlock = CallBlock->splitBasicBlock(SplitPt, "cond.end");
1540
1541 // Set up a builder to create a compare, conditional branch, and PHI.
1542 IRBuilder<> Builder(CountZeros->getContext());
1543 Builder.SetInsertPoint(StartBlock->getTerminator());
1544 Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc());
1545
1546 // Replace the unconditional branch that was created by the first split with
1547 // a compare against zero and a conditional branch.
1548 Value *Zero = Constant::getNullValue(Ty);
1549 Value *Cmp = Builder.CreateICmpEQ(CountZeros->getOperand(0), Zero, "cmpz");
1550 Builder.CreateCondBr(Cmp, EndBlock, CallBlock);
1551 StartBlock->getTerminator()->eraseFromParent();
1552
1553 // Create a PHI in the end block to select either the output of the intrinsic
1554 // or the bit width of the operand.
1555 Builder.SetInsertPoint(&EndBlock->front());
1556 PHINode *PN = Builder.CreatePHI(Ty, 2, "ctz");
1557 CountZeros->replaceAllUsesWith(PN);
1558 Value *BitWidth = Builder.getInt(APInt(SizeInBits, SizeInBits));
1559 PN->addIncoming(BitWidth, StartBlock);
1560 PN->addIncoming(CountZeros, CallBlock);
1561
1562 // We are explicitly handling the zero case, so we can set the intrinsic's
1563 // undefined zero argument to 'true'. This will also prevent reprocessing the
1564 // intrinsic; we only despeculate when a zero input is defined.
1565 CountZeros->setArgOperand(1, Builder.getTrue());
1566 ModifiedDT = true;
1567 return true;
1568}
1569
Sanjay Patel3b8974b2017-06-08 20:00:09 +00001570bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
Chris Lattner7a277142011-01-15 07:14:54 +00001571 BasicBlock *BB = CI->getParent();
Nadav Rotem465834c2012-07-24 10:51:42 +00001572
Chris Lattner7a277142011-01-15 07:14:54 +00001573 // Lower inline assembly if we can.
1574 // If we found an inline asm expession, and if the target knows how to
1575 // lower it to normal LLVM code, do so now.
1576 if (TLI && isa<InlineAsm>(CI->getCalledValue())) {
1577 if (TLI->ExpandInlineAsm(CI)) {
1578 // Avoid invalidating the iterator.
1579 CurInstIterator = BB->begin();
1580 // Avoid processing instructions out of order, which could cause
1581 // reuse before a value is defined.
1582 SunkAddrs.clear();
1583 return true;
1584 }
1585 // Sink address computing for memory operands into the block.
Sanjay Patelfc580a62015-09-21 23:03:16 +00001586 if (optimizeInlineAsmInst(CI))
Chris Lattner7a277142011-01-15 07:14:54 +00001587 return true;
1588 }
Nadav Rotem465834c2012-07-24 10:51:42 +00001589
John Brawn0dbcd652015-03-18 12:01:59 +00001590 // Align the pointer arguments to this call if the target thinks it's a good
1591 // idea
1592 unsigned MinSize, PrefAlign;
Mehdi Amini4fe37982015-07-07 18:45:17 +00001593 if (TLI && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {
John Brawn0dbcd652015-03-18 12:01:59 +00001594 for (auto &Arg : CI->arg_operands()) {
1595 // We want to align both objects whose address is used directly and
1596 // objects whose address is used in casts and GEPs, though it only makes
1597 // sense for GEPs if the offset is a multiple of the desired alignment and
1598 // if size - offset meets the size threshold.
1599 if (!Arg->getType()->isPointerTy())
1600 continue;
Mehdi Amini4fe37982015-07-07 18:45:17 +00001601 APInt Offset(DL->getPointerSizeInBits(
1602 cast<PointerType>(Arg->getType())->getAddressSpace()),
1603 0);
1604 Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset);
John Brawn0dbcd652015-03-18 12:01:59 +00001605 uint64_t Offset2 = Offset.getLimitedValue();
John Brawne8fd6c82015-04-13 10:47:39 +00001606 if ((Offset2 & (PrefAlign-1)) != 0)
1607 continue;
John Brawn0dbcd652015-03-18 12:01:59 +00001608 AllocaInst *AI;
Mehdi Amini4fe37982015-07-07 18:45:17 +00001609 if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlignment() < PrefAlign &&
1610 DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2)
John Brawn0dbcd652015-03-18 12:01:59 +00001611 AI->setAlignment(PrefAlign);
John Brawne8fd6c82015-04-13 10:47:39 +00001612 // Global variables can only be aligned if they are defined in this
1613 // object (i.e. they are uniquely initialized in this object), and
1614 // over-aligning global variables that have an explicit section is
1615 // forbidden.
1616 GlobalVariable *GV;
James Y Knightac03dca2016-01-15 16:33:06 +00001617 if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() &&
Tim Northover918f0502016-07-18 18:28:52 +00001618 GV->getPointerAlignment(*DL) < PrefAlign &&
Manuel Jacob5f6eaac2016-01-16 20:30:46 +00001619 DL->getTypeAllocSize(GV->getValueType()) >=
Mehdi Amini4fe37982015-07-07 18:45:17 +00001620 MinSize + Offset2)
John Brawne8fd6c82015-04-13 10:47:39 +00001621 GV->setAlignment(PrefAlign);
John Brawn0dbcd652015-03-18 12:01:59 +00001622 }
1623 // If this is a memcpy (or similar) then we may be able to improve the
1624 // alignment
1625 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) {
Mehdi Amini4fe37982015-07-07 18:45:17 +00001626 unsigned Align = getKnownAlignment(MI->getDest(), *DL);
John Brawn0dbcd652015-03-18 12:01:59 +00001627 if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
Mehdi Amini4fe37982015-07-07 18:45:17 +00001628 Align = std::min(Align, getKnownAlignment(MTI->getSource(), *DL));
Pete Cooper67cf9a72015-11-19 05:56:52 +00001629 if (Align > MI->getAlignment())
1630 MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), Align));
John Brawn0dbcd652015-03-18 12:01:59 +00001631 }
1632 }
1633
Philip Reamesac115ed2016-03-09 23:13:12 +00001634 // If we have a cold call site, try to sink addressing computation into the
1635 // cold block. This interacts with our handling for loads and stores to
1636 // ensure that we can fold all uses of a potential addressing computation
1637 // into their uses. TODO: generalize this to work over profiling data
1638 if (!OptSize && CI->hasFnAttr(Attribute::Cold))
1639 for (auto &Arg : CI->arg_operands()) {
1640 if (!Arg->getType()->isPointerTy())
1641 continue;
1642 unsigned AS = Arg->getType()->getPointerAddressSpace();
1643 return optimizeMemoryInst(CI, Arg, Arg->getType(), AS);
1644 }
Junmo Park6098cbb2016-03-11 07:05:32 +00001645
Eric Christopher4b7948e2010-03-11 02:41:03 +00001646 IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001647 if (II) {
1648 switch (II->getIntrinsicID()) {
1649 default: break;
1650 case Intrinsic::objectsize: {
1651 // Lower all uses of llvm.objectsize.*
George Burgess IV3f089142016-12-20 23:46:36 +00001652 ConstantInt *RetVal =
1653 lowerObjectSizeCall(II, *DL, TLInfo, /*MustSucceed=*/true);
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001654 // Substituting this can cause recursive simplifications, which can
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00001655 // invalidate our iterator. Use a WeakTrackingVH to hold onto it in case
1656 // this
Sanjoy Das2cbeb002017-04-26 16:37:05 +00001657 // happens.
Duncan P. N. Exon Smith7b269642016-02-21 19:37:45 +00001658 Value *CurValue = &*CurInstIterator;
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00001659 WeakTrackingVH IterHandle(CurValue);
Nadav Rotem465834c2012-07-24 10:51:42 +00001660
Sanjay Patel545a4562016-01-20 18:59:16 +00001661 replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr);
Chris Lattner1b93be52011-01-15 07:25:29 +00001662
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001663 // If the iterator instruction was recursively deleted, start over at the
1664 // start of the block.
Duncan P. N. Exon Smith7b269642016-02-21 19:37:45 +00001665 if (IterHandle != CurValue) {
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001666 CurInstIterator = BB->begin();
1667 SunkAddrs.clear();
1668 }
1669 return true;
Chris Lattner86d56c62011-01-18 20:53:04 +00001670 }
Ahmed Bougacha236f9042015-05-22 21:37:17 +00001671 case Intrinsic::aarch64_stlxr:
1672 case Intrinsic::aarch64_stxr: {
1673 ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0));
1674 if (!ExtVal || !ExtVal->hasOneUse() ||
1675 ExtVal->getParent() == CI->getParent())
1676 return false;
1677 // Sink a zext feeding stlxr/stxr before it, so it can be folded into it.
1678 ExtVal->moveBefore(CI);
Ahmed Bougachaf3299142015-06-17 20:44:32 +00001679 // Mark this instruction as "inserted by CGP", so that other
1680 // optimizations don't touch it.
1681 InsertedInsts.insert(ExtVal);
Ahmed Bougacha236f9042015-05-22 21:37:17 +00001682 return true;
1683 }
Piotr Padlewski6c15ec42015-09-15 18:32:14 +00001684 case Intrinsic::invariant_group_barrier:
1685 II->replaceAllUsesWith(II->getArgOperand(0));
1686 II->eraseFromParent();
1687 return true;
Sanjay Patel4699b8a2015-11-19 16:37:10 +00001688
1689 case Intrinsic::cttz:
1690 case Intrinsic::ctlz:
1691 // If counting zeros is expensive, try to avoid it.
1692 return despeculateCountZeros(II, TLI, DL, ModifiedDT);
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001693 }
Eric Christopher4b7948e2010-03-11 02:41:03 +00001694
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001695 if (TLI) {
1696 SmallVector<Value*, 2> PtrOps;
1697 Type *AccessTy;
Matt Arsenault1672b1b2017-02-08 07:09:03 +00001698 if (TLI->getAddrModeArguments(II, PtrOps, AccessTy))
1699 while (!PtrOps.empty()) {
1700 Value *PtrVal = PtrOps.pop_back_val();
1701 unsigned AS = PtrVal->getType()->getPointerAddressSpace();
1702 if (optimizeMemoryInst(II, PtrVal, AccessTy, AS))
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001703 return true;
Matt Arsenault1672b1b2017-02-08 07:09:03 +00001704 }
Elena Demikhovsky87700a72014-12-28 08:54:45 +00001705 }
Pete Cooper615fd892012-03-13 20:59:56 +00001706 }
1707
Eric Christopher4b7948e2010-03-11 02:41:03 +00001708 // From here on out we're working with named functions.
Craig Topperc0196b12014-04-14 00:51:57 +00001709 if (!CI->getCalledFunction()) return false;
Devang Patel0da52502011-05-26 21:51:06 +00001710
Benjamin Kramer7b88a492010-03-12 09:27:41 +00001711 // Lower all default uses of _chk calls. This is very similar
1712 // to what InstCombineCalls does, but here we are only lowering calls
Ahmed Bougachae03bef72015-01-12 17:22:43 +00001713 // to fortified library functions (e.g. __memcpy_chk) that have the default
1714 // "don't know" as the objectsize. Anything else should be left alone.
Mehdi Aminia28d91d2015-03-10 02:37:25 +00001715 FortifiedLibCallSimplifier Simplifier(TLInfo, true);
Ahmed Bougachae03bef72015-01-12 17:22:43 +00001716 if (Value *V = Simplifier.optimizeCall(CI)) {
1717 CI->replaceAllUsesWith(V);
1718 CI->eraseFromParent();
1719 return true;
1720 }
Zaara Syeda3a7578c2017-05-31 17:12:38 +00001721
Ahmed Bougachae03bef72015-01-12 17:22:43 +00001722 return false;
Eric Christopher4b7948e2010-03-11 02:41:03 +00001723}
Chris Lattner1b93be52011-01-15 07:25:29 +00001724
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001725/// Look for opportunities to duplicate return instructions to the predecessor
1726/// to enable tail call optimizations. The case it is currently looking for is:
Dmitri Gribenko2bc1d482012-09-13 12:34:29 +00001727/// @code
Evan Cheng0663f232011-03-21 01:19:09 +00001728/// bb0:
1729/// %tmp0 = tail call i32 @f0()
1730/// br label %return
1731/// bb1:
1732/// %tmp1 = tail call i32 @f1()
1733/// br label %return
1734/// bb2:
1735/// %tmp2 = tail call i32 @f2()
1736/// br label %return
1737/// return:
1738/// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
1739/// ret i32 %retval
Dmitri Gribenko2bc1d482012-09-13 12:34:29 +00001740/// @endcode
Evan Cheng0663f232011-03-21 01:19:09 +00001741///
1742/// =>
1743///
Dmitri Gribenko2bc1d482012-09-13 12:34:29 +00001744/// @code
Evan Cheng0663f232011-03-21 01:19:09 +00001745/// bb0:
1746/// %tmp0 = tail call i32 @f0()
1747/// ret i32 %tmp0
1748/// bb1:
1749/// %tmp1 = tail call i32 @f1()
1750/// ret i32 %tmp1
1751/// bb2:
1752/// %tmp2 = tail call i32 @f2()
1753/// ret i32 %tmp2
Dmitri Gribenko2bc1d482012-09-13 12:34:29 +00001754/// @endcode
Sanjay Patelfc580a62015-09-21 23:03:16 +00001755bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) {
Cameron Zwarich47e71752011-03-24 04:51:51 +00001756 if (!TLI)
1757 return false;
1758
Michael Kuperstein71321562016-09-07 20:29:49 +00001759 ReturnInst *RetI = dyn_cast<ReturnInst>(BB->getTerminator());
1760 if (!RetI)
Benjamin Kramer455fa352012-11-23 19:17:06 +00001761 return false;
1762
Craig Topperc0196b12014-04-14 00:51:57 +00001763 PHINode *PN = nullptr;
1764 BitCastInst *BCI = nullptr;
Michael Kuperstein71321562016-09-07 20:29:49 +00001765 Value *V = RetI->getReturnValue();
Evan Cheng249716e2012-07-27 21:21:26 +00001766 if (V) {
1767 BCI = dyn_cast<BitCastInst>(V);
1768 if (BCI)
1769 V = BCI->getOperand(0);
1770
1771 PN = dyn_cast<PHINode>(V);
1772 if (!PN)
1773 return false;
1774 }
Evan Cheng0663f232011-03-21 01:19:09 +00001775
Cameron Zwarich4649f172011-03-24 04:52:10 +00001776 if (PN && PN->getParent() != BB)
Cameron Zwarich0e331c02011-03-24 04:52:07 +00001777 return false;
Evan Cheng0663f232011-03-21 01:19:09 +00001778
Cameron Zwarich4649f172011-03-24 04:52:10 +00001779 // Make sure there are no instructions between the PHI and return, or that the
1780 // return is the first instruction in the block.
1781 if (PN) {
1782 BasicBlock::iterator BI = BB->begin();
1783 do { ++BI; } while (isa<DbgInfoIntrinsic>(BI));
Evan Cheng249716e2012-07-27 21:21:26 +00001784 if (&*BI == BCI)
1785 // Also skip over the bitcast.
1786 ++BI;
Michael Kuperstein71321562016-09-07 20:29:49 +00001787 if (&*BI != RetI)
Cameron Zwarich4649f172011-03-24 04:52:10 +00001788 return false;
1789 } else {
Cameron Zwarich74157ab2011-03-24 16:34:59 +00001790 BasicBlock::iterator BI = BB->begin();
1791 while (isa<DbgInfoIntrinsic>(BI)) ++BI;
Michael Kuperstein71321562016-09-07 20:29:49 +00001792 if (&*BI != RetI)
Cameron Zwarich4649f172011-03-24 04:52:10 +00001793 return false;
1794 }
Evan Cheng0663f232011-03-21 01:19:09 +00001795
Cameron Zwarich0e331c02011-03-24 04:52:07 +00001796 /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail
1797 /// call.
Michael Kupersteinf79af6f2016-09-08 00:48:37 +00001798 const Function *F = BB->getParent();
Cameron Zwarich0e331c02011-03-24 04:52:07 +00001799 SmallVector<CallInst*, 4> TailCalls;
Cameron Zwarich4649f172011-03-24 04:52:10 +00001800 if (PN) {
1801 for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) {
1802 CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I));
1803 // Make sure the phi value is indeed produced by the tail call.
1804 if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) &&
Michael Kupersteinf79af6f2016-09-08 00:48:37 +00001805 TLI->mayBeEmittedAsTailCall(CI) &&
1806 attributesPermitTailCall(F, CI, RetI, *TLI))
Cameron Zwarich4649f172011-03-24 04:52:10 +00001807 TailCalls.push_back(CI);
1808 }
1809 } else {
1810 SmallPtrSet<BasicBlock*, 4> VisitedBBs;
Duncan P. N. Exon Smith6c990152014-07-21 17:06:51 +00001811 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) {
David Blaikie70573dc2014-11-19 07:49:26 +00001812 if (!VisitedBBs.insert(*PI).second)
Cameron Zwarich4649f172011-03-24 04:52:10 +00001813 continue;
1814
Duncan P. N. Exon Smith6c990152014-07-21 17:06:51 +00001815 BasicBlock::InstListType &InstList = (*PI)->getInstList();
Cameron Zwarich4649f172011-03-24 04:52:10 +00001816 BasicBlock::InstListType::reverse_iterator RI = InstList.rbegin();
1817 BasicBlock::InstListType::reverse_iterator RE = InstList.rend();
Cameron Zwarich74157ab2011-03-24 16:34:59 +00001818 do { ++RI; } while (RI != RE && isa<DbgInfoIntrinsic>(&*RI));
1819 if (RI == RE)
Cameron Zwarich4649f172011-03-24 04:52:10 +00001820 continue;
Cameron Zwarich74157ab2011-03-24 16:34:59 +00001821
Cameron Zwarich4649f172011-03-24 04:52:10 +00001822 CallInst *CI = dyn_cast<CallInst>(&*RI);
Michael Kupersteinf79af6f2016-09-08 00:48:37 +00001823 if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI) &&
1824 attributesPermitTailCall(F, CI, RetI, *TLI))
Cameron Zwarich4649f172011-03-24 04:52:10 +00001825 TailCalls.push_back(CI);
1826 }
Evan Cheng0663f232011-03-21 01:19:09 +00001827 }
1828
Cameron Zwarich0e331c02011-03-24 04:52:07 +00001829 bool Changed = false;
1830 for (unsigned i = 0, e = TailCalls.size(); i != e; ++i) {
1831 CallInst *CI = TailCalls[i];
1832 CallSite CS(CI);
1833
1834 // Conservatively require the attributes of the call to match those of the
1835 // return. Ignore noalias because it doesn't affect the call sequence.
Reid Klecknerb5180542017-03-21 16:57:19 +00001836 AttributeList CalleeAttrs = CS.getAttributes();
1837 if (AttrBuilder(CalleeAttrs, AttributeList::ReturnIndex)
1838 .removeAttribute(Attribute::NoAlias) !=
1839 AttrBuilder(CalleeAttrs, AttributeList::ReturnIndex)
1840 .removeAttribute(Attribute::NoAlias))
Cameron Zwarich0e331c02011-03-24 04:52:07 +00001841 continue;
1842
1843 // Make sure the call instruction is followed by an unconditional branch to
1844 // the return block.
1845 BasicBlock *CallBB = CI->getParent();
1846 BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator());
1847 if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB)
1848 continue;
1849
1850 // Duplicate the return into CallBB.
Michael Kuperstein71321562016-09-07 20:29:49 +00001851 (void)FoldReturnIntoUncondBranch(RetI, BB, CallBB);
Devang Patel8f606d72011-03-24 15:35:25 +00001852 ModifiedDT = Changed = true;
Cameron Zwarich0e331c02011-03-24 04:52:07 +00001853 ++NumRetsDup;
1854 }
1855
1856 // If we eliminated all predecessors of the block, delete the block now.
Evan Cheng64a223a2012-09-28 23:58:57 +00001857 if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB))
Cameron Zwarich0e331c02011-03-24 04:52:07 +00001858 BB->eraseFromParent();
1859
1860 return Changed;
Evan Cheng0663f232011-03-21 01:19:09 +00001861}
1862
Chris Lattner728f9022008-11-25 07:09:13 +00001863//===----------------------------------------------------------------------===//
Chris Lattner728f9022008-11-25 07:09:13 +00001864// Memory Optimization
1865//===----------------------------------------------------------------------===//
1866
Chandler Carruthc8925912013-01-05 02:09:22 +00001867namespace {
1868
Sanjay Patel4ac6b112015-09-21 22:47:23 +00001869/// This is an extended version of TargetLowering::AddrMode
Chandler Carruthc8925912013-01-05 02:09:22 +00001870/// which holds actual Value*'s for register values.
Chandler Carruth95f83e02013-01-07 15:14:13 +00001871struct ExtAddrMode : public TargetLowering::AddrMode {
Eugene Zelenko900b6332017-08-29 22:32:07 +00001872 Value *BaseReg = nullptr;
1873 Value *ScaledReg = nullptr;
John Brawn736bf002017-10-03 13:08:22 +00001874 Value *OriginalValue = nullptr;
1875
1876 enum FieldName {
1877 NoField = 0x00,
1878 BaseRegField = 0x01,
1879 BaseGVField = 0x02,
1880 BaseOffsField = 0x04,
1881 ScaledRegField = 0x08,
1882 ScaleField = 0x10,
1883 MultipleFields = 0xff
1884 };
Eugene Zelenko900b6332017-08-29 22:32:07 +00001885
1886 ExtAddrMode() = default;
1887
Chandler Carruthc8925912013-01-05 02:09:22 +00001888 void print(raw_ostream &OS) const;
1889 void dump() const;
Stephen Lin837bba12013-07-15 17:55:02 +00001890
John Brawn736bf002017-10-03 13:08:22 +00001891 FieldName compare(const ExtAddrMode &other) {
1892 // First check that the types are the same on each field, as differing types
1893 // is something we can't cope with later on.
1894 if (BaseReg && other.BaseReg &&
1895 BaseReg->getType() != other.BaseReg->getType())
1896 return MultipleFields;
1897 if (BaseGV && other.BaseGV &&
1898 BaseGV->getType() != other.BaseGV->getType())
1899 return MultipleFields;
1900 if (ScaledReg && other.ScaledReg &&
1901 ScaledReg->getType() != other.ScaledReg->getType())
1902 return MultipleFields;
1903
1904 // Check each field to see if it differs.
1905 unsigned Result = NoField;
1906 if (BaseReg != other.BaseReg)
1907 Result |= BaseRegField;
1908 if (BaseGV != other.BaseGV)
1909 Result |= BaseGVField;
1910 if (BaseOffs != other.BaseOffs)
1911 Result |= BaseOffsField;
1912 if (ScaledReg != other.ScaledReg)
1913 Result |= ScaledRegField;
1914 // Don't count 0 as being a different scale, because that actually means
1915 // unscaled (which will already be counted by having no ScaledReg).
1916 if (Scale && other.Scale && Scale != other.Scale)
1917 Result |= ScaleField;
1918
1919 if (countPopulation(Result) > 1)
1920 return MultipleFields;
1921 else
1922 return static_cast<FieldName>(Result);
1923 }
1924
John Brawn4b476482017-11-27 11:29:15 +00001925 // An AddrMode is trivial if it involves no calculation i.e. it is just a base
1926 // with no offset.
John Brawn736bf002017-10-03 13:08:22 +00001927 bool isTrivial() {
John Brawn4b476482017-11-27 11:29:15 +00001928 // An AddrMode is (BaseGV + BaseReg + BaseOffs + ScaleReg * Scale) so it is
1929 // trivial if at most one of these terms is nonzero, except that BaseGV and
1930 // BaseReg both being zero actually means a null pointer value, which we
1931 // consider to be 'non-zero' here.
1932 return !BaseOffs && !Scale && !(BaseGV && BaseReg);
Chandler Carruthc8925912013-01-05 02:09:22 +00001933 }
John Brawn70cdb5b2017-11-24 14:10:45 +00001934
1935 Value *GetFieldAsValue(FieldName Field, Type *IntPtrTy) {
1936 switch (Field) {
1937 default:
1938 return nullptr;
1939 case BaseRegField:
1940 return BaseReg;
1941 case BaseGVField:
1942 return BaseGV;
1943 case ScaledRegField:
1944 return ScaledReg;
1945 case BaseOffsField:
1946 return ConstantInt::get(IntPtrTy, BaseOffs);
1947 }
1948 }
1949
1950 void SetCombinedField(FieldName Field, Value *V,
1951 const SmallVectorImpl<ExtAddrMode> &AddrModes) {
1952 switch (Field) {
1953 default:
1954 llvm_unreachable("Unhandled fields are expected to be rejected earlier");
1955 break;
1956 case ExtAddrMode::BaseRegField:
1957 BaseReg = V;
1958 break;
1959 case ExtAddrMode::BaseGVField:
1960 // A combined BaseGV is an Instruction, not a GlobalValue, so it goes
1961 // in the BaseReg field.
1962 assert(BaseReg == nullptr);
1963 BaseReg = V;
1964 BaseGV = nullptr;
1965 break;
1966 case ExtAddrMode::ScaledRegField:
1967 ScaledReg = V;
1968 // If we have a mix of scaled and unscaled addrmodes then we want scale
1969 // to be the scale and not zero.
1970 if (!Scale)
1971 for (const ExtAddrMode &AM : AddrModes)
1972 if (AM.Scale) {
1973 Scale = AM.Scale;
1974 break;
1975 }
1976 break;
1977 case ExtAddrMode::BaseOffsField:
1978 // The offset is no longer a constant, so it goes in ScaledReg with a
1979 // scale of 1.
1980 assert(ScaledReg == nullptr);
1981 ScaledReg = V;
1982 Scale = 1;
1983 BaseOffs = 0;
1984 break;
1985 }
1986 }
Chandler Carruthc8925912013-01-05 02:09:22 +00001987};
1988
Eugene Zelenko900b6332017-08-29 22:32:07 +00001989} // end anonymous namespace
1990
Eli Friedmanc1f1f852013-09-10 23:09:24 +00001991#ifndef NDEBUG
1992static inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) {
1993 AM.print(OS);
1994 return OS;
1995}
1996#endif
1997
Aaron Ballman615eb472017-10-15 14:32:27 +00001998#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Chandler Carruthc8925912013-01-05 02:09:22 +00001999void ExtAddrMode::print(raw_ostream &OS) const {
2000 bool NeedPlus = false;
2001 OS << "[";
2002 if (BaseGV) {
2003 OS << (NeedPlus ? " + " : "")
2004 << "GV:";
Chandler Carruthd48cdbf2014-01-09 02:29:41 +00002005 BaseGV->printAsOperand(OS, /*PrintType=*/false);
Chandler Carruthc8925912013-01-05 02:09:22 +00002006 NeedPlus = true;
2007 }
2008
Richard Trieuc0f91212014-05-30 03:15:17 +00002009 if (BaseOffs) {
2010 OS << (NeedPlus ? " + " : "")
2011 << BaseOffs;
2012 NeedPlus = true;
2013 }
Chandler Carruthc8925912013-01-05 02:09:22 +00002014
2015 if (BaseReg) {
2016 OS << (NeedPlus ? " + " : "")
2017 << "Base:";
Chandler Carruthd48cdbf2014-01-09 02:29:41 +00002018 BaseReg->printAsOperand(OS, /*PrintType=*/false);
Chandler Carruthc8925912013-01-05 02:09:22 +00002019 NeedPlus = true;
2020 }
2021 if (Scale) {
2022 OS << (NeedPlus ? " + " : "")
2023 << Scale << "*";
Chandler Carruthd48cdbf2014-01-09 02:29:41 +00002024 ScaledReg->printAsOperand(OS, /*PrintType=*/false);
Chandler Carruthc8925912013-01-05 02:09:22 +00002025 }
2026
2027 OS << ']';
2028}
2029
Yaron Kereneb2a2542016-01-29 20:50:44 +00002030LLVM_DUMP_METHOD void ExtAddrMode::dump() const {
Chandler Carruthc8925912013-01-05 02:09:22 +00002031 print(dbgs());
2032 dbgs() << '\n';
2033}
2034#endif
2035
Eugene Zelenko900b6332017-08-29 22:32:07 +00002036namespace {
2037
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002038/// \brief This class provides transaction based operation on the IR.
2039/// Every change made through this class is recorded in the internal state and
2040/// can be undone (rollback) until commit is called.
2041class TypePromotionTransaction {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002042 /// \brief This represents the common interface of the individual transaction.
2043 /// Each class implements the logic for doing one specific modification on
2044 /// the IR via the TypePromotionTransaction.
2045 class TypePromotionAction {
2046 protected:
2047 /// The Instruction modified.
2048 Instruction *Inst;
2049
2050 public:
2051 /// \brief Constructor of the action.
2052 /// The constructor performs the related action on the IR.
2053 TypePromotionAction(Instruction *Inst) : Inst(Inst) {}
2054
Eugene Zelenko900b6332017-08-29 22:32:07 +00002055 virtual ~TypePromotionAction() = default;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002056
2057 /// \brief Undo the modification done by this action.
2058 /// When this method is called, the IR must be in the same state as it was
2059 /// before this action was applied.
2060 /// \pre Undoing the action works if and only if the IR is in the exact same
2061 /// state as it was directly after this action was applied.
2062 virtual void undo() = 0;
2063
2064 /// \brief Advocate every change made by this action.
2065 /// When the results on the IR of the action are to be kept, it is important
2066 /// to call this function, otherwise hidden information may be kept forever.
2067 virtual void commit() {
2068 // Nothing to be done, this action is not doing anything.
2069 }
2070 };
2071
2072 /// \brief Utility to remember the position of an instruction.
2073 class InsertionHandler {
2074 /// Position of an instruction.
2075 /// Either an instruction:
2076 /// - Is the first in a basic block: BB is used.
2077 /// - Has a previous instructon: PrevInst is used.
2078 union {
2079 Instruction *PrevInst;
2080 BasicBlock *BB;
2081 } Point;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002082
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002083 /// Remember whether or not the instruction had a previous instruction.
2084 bool HasPrevInstruction;
2085
2086 public:
2087 /// \brief Record the position of \p Inst.
2088 InsertionHandler(Instruction *Inst) {
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00002089 BasicBlock::iterator It = Inst->getIterator();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002090 HasPrevInstruction = (It != (Inst->getParent()->begin()));
2091 if (HasPrevInstruction)
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00002092 Point.PrevInst = &*--It;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002093 else
2094 Point.BB = Inst->getParent();
2095 }
2096
2097 /// \brief Insert \p Inst at the recorded position.
2098 void insert(Instruction *Inst) {
2099 if (HasPrevInstruction) {
2100 if (Inst->getParent())
2101 Inst->removeFromParent();
2102 Inst->insertAfter(Point.PrevInst);
2103 } else {
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00002104 Instruction *Position = &*Point.BB->getFirstInsertionPt();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002105 if (Inst->getParent())
2106 Inst->moveBefore(Position);
2107 else
2108 Inst->insertBefore(Position);
2109 }
2110 }
2111 };
2112
2113 /// \brief Move an instruction before another.
2114 class InstructionMoveBefore : public TypePromotionAction {
2115 /// Original position of the instruction.
2116 InsertionHandler Position;
2117
2118 public:
2119 /// \brief Move \p Inst before \p Before.
2120 InstructionMoveBefore(Instruction *Inst, Instruction *Before)
2121 : TypePromotionAction(Inst), Position(Inst) {
2122 DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before << "\n");
2123 Inst->moveBefore(Before);
2124 }
2125
2126 /// \brief Move the instruction back to its original position.
Craig Topper4584cd52014-03-07 09:26:03 +00002127 void undo() override {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002128 DEBUG(dbgs() << "Undo: moveBefore: " << *Inst << "\n");
2129 Position.insert(Inst);
2130 }
2131 };
2132
2133 /// \brief Set the operand of an instruction with a new value.
2134 class OperandSetter : public TypePromotionAction {
2135 /// Original operand of the instruction.
2136 Value *Origin;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002137
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002138 /// Index of the modified instruction.
2139 unsigned Idx;
2140
2141 public:
2142 /// \brief Set \p Idx operand of \p Inst with \p NewVal.
2143 OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal)
2144 : TypePromotionAction(Inst), Idx(Idx) {
2145 DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n"
2146 << "for:" << *Inst << "\n"
2147 << "with:" << *NewVal << "\n");
2148 Origin = Inst->getOperand(Idx);
2149 Inst->setOperand(Idx, NewVal);
2150 }
2151
2152 /// \brief Restore the original value of the instruction.
Craig Topper4584cd52014-03-07 09:26:03 +00002153 void undo() override {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002154 DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n"
2155 << "for: " << *Inst << "\n"
2156 << "with: " << *Origin << "\n");
2157 Inst->setOperand(Idx, Origin);
2158 }
2159 };
2160
2161 /// \brief Hide the operands of an instruction.
2162 /// Do as if this instruction was not using any of its operands.
2163 class OperandsHider : public TypePromotionAction {
2164 /// The list of original operands.
2165 SmallVector<Value *, 4> OriginalValues;
2166
2167 public:
2168 /// \brief Remove \p Inst from the uses of the operands of \p Inst.
2169 OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) {
2170 DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n");
2171 unsigned NumOpnds = Inst->getNumOperands();
2172 OriginalValues.reserve(NumOpnds);
2173 for (unsigned It = 0; It < NumOpnds; ++It) {
2174 // Save the current operand.
2175 Value *Val = Inst->getOperand(It);
2176 OriginalValues.push_back(Val);
2177 // Set a dummy one.
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00002178 // We could use OperandSetter here, but that would imply an overhead
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002179 // that we are not willing to pay.
2180 Inst->setOperand(It, UndefValue::get(Val->getType()));
2181 }
2182 }
2183
2184 /// \brief Restore the original list of uses.
Craig Topper4584cd52014-03-07 09:26:03 +00002185 void undo() override {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002186 DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n");
2187 for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It)
2188 Inst->setOperand(It, OriginalValues[It]);
2189 }
2190 };
2191
2192 /// \brief Build a truncate instruction.
2193 class TruncBuilder : public TypePromotionAction {
Quentin Colombetac55b152014-09-16 22:36:07 +00002194 Value *Val;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002195
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002196 public:
2197 /// \brief Build a truncate instruction of \p Opnd producing a \p Ty
2198 /// result.
2199 /// trunc Opnd to Ty.
2200 TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) {
2201 IRBuilder<> Builder(Opnd);
Quentin Colombetac55b152014-09-16 22:36:07 +00002202 Val = Builder.CreateTrunc(Opnd, Ty, "promoted");
2203 DEBUG(dbgs() << "Do: TruncBuilder: " << *Val << "\n");
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002204 }
2205
Quentin Colombetac55b152014-09-16 22:36:07 +00002206 /// \brief Get the built value.
2207 Value *getBuiltValue() { return Val; }
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002208
2209 /// \brief Remove the built instruction.
Craig Topper4584cd52014-03-07 09:26:03 +00002210 void undo() override {
Quentin Colombetac55b152014-09-16 22:36:07 +00002211 DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val << "\n");
2212 if (Instruction *IVal = dyn_cast<Instruction>(Val))
2213 IVal->eraseFromParent();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002214 }
2215 };
2216
2217 /// \brief Build a sign extension instruction.
2218 class SExtBuilder : public TypePromotionAction {
Quentin Colombetac55b152014-09-16 22:36:07 +00002219 Value *Val;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002220
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002221 public:
2222 /// \brief Build a sign extension instruction of \p Opnd producing a \p Ty
2223 /// result.
2224 /// sext Opnd to Ty.
2225 SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
Quentin Colombetac55b152014-09-16 22:36:07 +00002226 : TypePromotionAction(InsertPt) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002227 IRBuilder<> Builder(InsertPt);
Quentin Colombetac55b152014-09-16 22:36:07 +00002228 Val = Builder.CreateSExt(Opnd, Ty, "promoted");
2229 DEBUG(dbgs() << "Do: SExtBuilder: " << *Val << "\n");
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002230 }
2231
Quentin Colombetac55b152014-09-16 22:36:07 +00002232 /// \brief Get the built value.
2233 Value *getBuiltValue() { return Val; }
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002234
2235 /// \brief Remove the built instruction.
Craig Topper4584cd52014-03-07 09:26:03 +00002236 void undo() override {
Quentin Colombetac55b152014-09-16 22:36:07 +00002237 DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val << "\n");
2238 if (Instruction *IVal = dyn_cast<Instruction>(Val))
2239 IVal->eraseFromParent();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002240 }
2241 };
2242
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002243 /// \brief Build a zero extension instruction.
2244 class ZExtBuilder : public TypePromotionAction {
Quentin Colombetac55b152014-09-16 22:36:07 +00002245 Value *Val;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002246
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002247 public:
2248 /// \brief Build a zero extension instruction of \p Opnd producing a \p Ty
2249 /// result.
2250 /// zext Opnd to Ty.
2251 ZExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)
Quentin Colombetac55b152014-09-16 22:36:07 +00002252 : TypePromotionAction(InsertPt) {
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002253 IRBuilder<> Builder(InsertPt);
Quentin Colombetac55b152014-09-16 22:36:07 +00002254 Val = Builder.CreateZExt(Opnd, Ty, "promoted");
2255 DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val << "\n");
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002256 }
2257
Quentin Colombetac55b152014-09-16 22:36:07 +00002258 /// \brief Get the built value.
2259 Value *getBuiltValue() { return Val; }
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002260
2261 /// \brief Remove the built instruction.
2262 void undo() override {
Quentin Colombetac55b152014-09-16 22:36:07 +00002263 DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val << "\n");
2264 if (Instruction *IVal = dyn_cast<Instruction>(Val))
2265 IVal->eraseFromParent();
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002266 }
2267 };
2268
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002269 /// \brief Mutate an instruction to another type.
2270 class TypeMutator : public TypePromotionAction {
2271 /// Record the original type.
2272 Type *OrigTy;
2273
2274 public:
2275 /// \brief Mutate the type of \p Inst into \p NewTy.
2276 TypeMutator(Instruction *Inst, Type *NewTy)
2277 : TypePromotionAction(Inst), OrigTy(Inst->getType()) {
2278 DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy
2279 << "\n");
2280 Inst->mutateType(NewTy);
2281 }
2282
2283 /// \brief Mutate the instruction back to its original type.
Craig Topper4584cd52014-03-07 09:26:03 +00002284 void undo() override {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002285 DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy
2286 << "\n");
2287 Inst->mutateType(OrigTy);
2288 }
2289 };
2290
2291 /// \brief Replace the uses of an instruction by another instruction.
2292 class UsesReplacer : public TypePromotionAction {
2293 /// Helper structure to keep track of the replaced uses.
2294 struct InstructionAndIdx {
2295 /// The instruction using the instruction.
2296 Instruction *Inst;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002297
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002298 /// The index where this instruction is used for Inst.
2299 unsigned Idx;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002300
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002301 InstructionAndIdx(Instruction *Inst, unsigned Idx)
2302 : Inst(Inst), Idx(Idx) {}
2303 };
2304
2305 /// Keep track of the original uses (pair Instruction, Index).
2306 SmallVector<InstructionAndIdx, 4> OriginalUses;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002307
2308 using use_iterator = SmallVectorImpl<InstructionAndIdx>::iterator;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002309
2310 public:
2311 /// \brief Replace all the use of \p Inst by \p New.
2312 UsesReplacer(Instruction *Inst, Value *New) : TypePromotionAction(Inst) {
2313 DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New
2314 << "\n");
2315 // Record the original uses.
Chandler Carruthcdf47882014-03-09 03:16:01 +00002316 for (Use &U : Inst->uses()) {
2317 Instruction *UserI = cast<Instruction>(U.getUser());
2318 OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo()));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002319 }
2320 // Now, we can replace the uses.
2321 Inst->replaceAllUsesWith(New);
2322 }
2323
2324 /// \brief Reassign the original uses of Inst to Inst.
Craig Topper4584cd52014-03-07 09:26:03 +00002325 void undo() override {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002326 DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n");
2327 for (use_iterator UseIt = OriginalUses.begin(),
2328 EndIt = OriginalUses.end();
2329 UseIt != EndIt; ++UseIt) {
2330 UseIt->Inst->setOperand(UseIt->Idx, Inst);
2331 }
2332 }
2333 };
2334
2335 /// \brief Remove an instruction from the IR.
2336 class InstructionRemover : public TypePromotionAction {
2337 /// Original position of the instruction.
2338 InsertionHandler Inserter;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002339
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002340 /// Helper structure to hide all the link to the instruction. In other
2341 /// words, this helps to do as if the instruction was removed.
2342 OperandsHider Hider;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002343
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002344 /// Keep track of the uses replaced, if any.
Eugene Zelenko900b6332017-08-29 22:32:07 +00002345 UsesReplacer *Replacer = nullptr;
2346
Jun Bum Limdee55652017-04-03 19:20:07 +00002347 /// Keep track of instructions removed.
2348 SetOfInstrs &RemovedInsts;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002349
2350 public:
2351 /// \brief Remove all reference of \p Inst and optinally replace all its
2352 /// uses with New.
Jun Bum Limdee55652017-04-03 19:20:07 +00002353 /// \p RemovedInsts Keep track of the instructions removed by this Action.
Craig Topperc0196b12014-04-14 00:51:57 +00002354 /// \pre If !Inst->use_empty(), then New != nullptr
Jun Bum Limdee55652017-04-03 19:20:07 +00002355 InstructionRemover(Instruction *Inst, SetOfInstrs &RemovedInsts,
2356 Value *New = nullptr)
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002357 : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst),
Eugene Zelenko900b6332017-08-29 22:32:07 +00002358 RemovedInsts(RemovedInsts) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002359 if (New)
2360 Replacer = new UsesReplacer(Inst, New);
2361 DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n");
Jun Bum Limdee55652017-04-03 19:20:07 +00002362 RemovedInsts.insert(Inst);
2363 /// The instructions removed here will be freed after completing
2364 /// optimizeBlock() for all blocks as we need to keep track of the
2365 /// removed instructions during promotion.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002366 Inst->removeFromParent();
2367 }
2368
Alexander Kornienkof817c1c2015-04-11 02:11:45 +00002369 ~InstructionRemover() override { delete Replacer; }
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002370
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002371 /// \brief Resurrect the instruction and reassign it to the proper uses if
2372 /// new value was provided when build this action.
Craig Topper4584cd52014-03-07 09:26:03 +00002373 void undo() override {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002374 DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n");
2375 Inserter.insert(Inst);
2376 if (Replacer)
2377 Replacer->undo();
2378 Hider.undo();
Jun Bum Limdee55652017-04-03 19:20:07 +00002379 RemovedInsts.erase(Inst);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002380 }
2381 };
2382
2383public:
2384 /// Restoration point.
2385 /// The restoration point is a pointer to an action instead of an iterator
2386 /// because the iterator may be invalidated but not the pointer.
Eugene Zelenko900b6332017-08-29 22:32:07 +00002387 using ConstRestorationPt = const TypePromotionAction *;
Jun Bum Limdee55652017-04-03 19:20:07 +00002388
2389 TypePromotionTransaction(SetOfInstrs &RemovedInsts)
2390 : RemovedInsts(RemovedInsts) {}
2391
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002392 /// Advocate every changes made in that transaction.
2393 void commit();
Eugene Zelenko900b6332017-08-29 22:32:07 +00002394
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002395 /// Undo all the changes made after the given point.
2396 void rollback(ConstRestorationPt Point);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002397
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002398 /// Get the current restoration point.
2399 ConstRestorationPt getRestorationPoint() const;
2400
2401 /// \name API for IR modification with state keeping to support rollback.
2402 /// @{
2403 /// Same as Instruction::setOperand.
2404 void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002405
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002406 /// Same as Instruction::eraseFromParent.
Craig Topperc0196b12014-04-14 00:51:57 +00002407 void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002408
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002409 /// Same as Value::replaceAllUsesWith.
2410 void replaceAllUsesWith(Instruction *Inst, Value *New);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002411
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002412 /// Same as Value::mutateType.
2413 void mutateType(Instruction *Inst, Type *NewTy);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002414
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002415 /// Same as IRBuilder::createTrunc.
Quentin Colombetac55b152014-09-16 22:36:07 +00002416 Value *createTrunc(Instruction *Opnd, Type *Ty);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002417
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002418 /// Same as IRBuilder::createSExt.
Quentin Colombetac55b152014-09-16 22:36:07 +00002419 Value *createSExt(Instruction *Inst, Value *Opnd, Type *Ty);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002420
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002421 /// Same as IRBuilder::createZExt.
Quentin Colombetac55b152014-09-16 22:36:07 +00002422 Value *createZExt(Instruction *Inst, Value *Opnd, Type *Ty);
Eugene Zelenko900b6332017-08-29 22:32:07 +00002423
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002424 /// Same as Instruction::moveBefore.
2425 void moveBefore(Instruction *Inst, Instruction *Before);
2426 /// @}
2427
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002428private:
2429 /// The ordered list of actions made so far.
David Blaikie7620b312014-04-15 06:17:44 +00002430 SmallVector<std::unique_ptr<TypePromotionAction>, 16> Actions;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002431
2432 using CommitPt = SmallVectorImpl<std::unique_ptr<TypePromotionAction>>::iterator;
2433
Jun Bum Limdee55652017-04-03 19:20:07 +00002434 SetOfInstrs &RemovedInsts;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002435};
2436
Eugene Zelenko900b6332017-08-29 22:32:07 +00002437} // end anonymous namespace
2438
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002439void TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx,
2440 Value *NewVal) {
Eugene Zelenko900b6332017-08-29 22:32:07 +00002441 Actions.push_back(llvm::make_unique<TypePromotionTransaction::OperandSetter>(
2442 Inst, Idx, NewVal));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002443}
2444
2445void TypePromotionTransaction::eraseInstruction(Instruction *Inst,
2446 Value *NewVal) {
2447 Actions.push_back(
Eugene Zelenko900b6332017-08-29 22:32:07 +00002448 llvm::make_unique<TypePromotionTransaction::InstructionRemover>(
2449 Inst, RemovedInsts, NewVal));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002450}
2451
2452void TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst,
2453 Value *New) {
Eugene Zelenko900b6332017-08-29 22:32:07 +00002454 Actions.push_back(
2455 llvm::make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002456}
2457
2458void TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) {
Eugene Zelenko900b6332017-08-29 22:32:07 +00002459 Actions.push_back(
2460 llvm::make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002461}
2462
Quentin Colombetac55b152014-09-16 22:36:07 +00002463Value *TypePromotionTransaction::createTrunc(Instruction *Opnd,
2464 Type *Ty) {
David Blaikie7620b312014-04-15 06:17:44 +00002465 std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty));
Quentin Colombetac55b152014-09-16 22:36:07 +00002466 Value *Val = Ptr->getBuiltValue();
David Blaikie7620b312014-04-15 06:17:44 +00002467 Actions.push_back(std::move(Ptr));
Quentin Colombetac55b152014-09-16 22:36:07 +00002468 return Val;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002469}
2470
Quentin Colombetac55b152014-09-16 22:36:07 +00002471Value *TypePromotionTransaction::createSExt(Instruction *Inst,
2472 Value *Opnd, Type *Ty) {
David Blaikie7620b312014-04-15 06:17:44 +00002473 std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty));
Quentin Colombetac55b152014-09-16 22:36:07 +00002474 Value *Val = Ptr->getBuiltValue();
David Blaikie7620b312014-04-15 06:17:44 +00002475 Actions.push_back(std::move(Ptr));
Quentin Colombetac55b152014-09-16 22:36:07 +00002476 return Val;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002477}
2478
Quentin Colombetac55b152014-09-16 22:36:07 +00002479Value *TypePromotionTransaction::createZExt(Instruction *Inst,
2480 Value *Opnd, Type *Ty) {
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002481 std::unique_ptr<ZExtBuilder> Ptr(new ZExtBuilder(Inst, Opnd, Ty));
Quentin Colombetac55b152014-09-16 22:36:07 +00002482 Value *Val = Ptr->getBuiltValue();
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002483 Actions.push_back(std::move(Ptr));
Quentin Colombetac55b152014-09-16 22:36:07 +00002484 return Val;
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00002485}
2486
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002487void TypePromotionTransaction::moveBefore(Instruction *Inst,
2488 Instruction *Before) {
2489 Actions.push_back(
Eugene Zelenko900b6332017-08-29 22:32:07 +00002490 llvm::make_unique<TypePromotionTransaction::InstructionMoveBefore>(
2491 Inst, Before));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002492}
2493
2494TypePromotionTransaction::ConstRestorationPt
2495TypePromotionTransaction::getRestorationPoint() const {
David Blaikie7620b312014-04-15 06:17:44 +00002496 return !Actions.empty() ? Actions.back().get() : nullptr;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002497}
2498
2499void TypePromotionTransaction::commit() {
2500 for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt;
David Blaikie7620b312014-04-15 06:17:44 +00002501 ++It)
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002502 (*It)->commit();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002503 Actions.clear();
2504}
2505
2506void TypePromotionTransaction::rollback(
2507 TypePromotionTransaction::ConstRestorationPt Point) {
David Blaikie7620b312014-04-15 06:17:44 +00002508 while (!Actions.empty() && Point != Actions.back().get()) {
2509 std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002510 Curr->undo();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002511 }
2512}
2513
Eugene Zelenko900b6332017-08-29 22:32:07 +00002514namespace {
2515
Chandler Carruthc8925912013-01-05 02:09:22 +00002516/// \brief A helper class for matching addressing modes.
2517///
2518/// This encapsulates the logic for matching the target-legal addressing modes.
2519class AddressingModeMatcher {
2520 SmallVectorImpl<Instruction*> &AddrModeInsts;
2521 const TargetLowering &TLI;
Igor Laevsky3be81ba2017-02-07 13:27:20 +00002522 const TargetRegisterInfo &TRI;
Mehdi Amini4fe37982015-07-07 18:45:17 +00002523 const DataLayout &DL;
Chandler Carruthc8925912013-01-05 02:09:22 +00002524
2525 /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
2526 /// the memory instruction that we're computing this address for.
2527 Type *AccessTy;
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00002528 unsigned AddrSpace;
Chandler Carruthc8925912013-01-05 02:09:22 +00002529 Instruction *MemoryInst;
Stephen Lin837bba12013-07-15 17:55:02 +00002530
Sanjay Patel4ac6b112015-09-21 22:47:23 +00002531 /// This is the addressing mode that we're building up. This is
Chandler Carruthc8925912013-01-05 02:09:22 +00002532 /// part of the return value of this addressing mode matching stuff.
2533 ExtAddrMode &AddrMode;
Stephen Lin837bba12013-07-15 17:55:02 +00002534
Ahmed Bougachaf3299142015-06-17 20:44:32 +00002535 /// The instructions inserted by other CodeGenPrepare optimizations.
2536 const SetOfInstrs &InsertedInsts;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002537
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002538 /// A map from the instructions to their type before promotion.
2539 InstrToOrigTy &PromotedInsts;
Eugene Zelenko900b6332017-08-29 22:32:07 +00002540
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002541 /// The ongoing transaction where every action should be registered.
2542 TypePromotionTransaction &TPT;
2543
Sanjay Patel4ac6b112015-09-21 22:47:23 +00002544 /// This is set to true when we should not do profitability checks.
2545 /// When true, IsProfitableToFoldIntoAddressingMode always returns true.
Chandler Carruthc8925912013-01-05 02:09:22 +00002546 bool IgnoreProfitability;
Stephen Lin837bba12013-07-15 17:55:02 +00002547
Eric Christopherd75c00c2015-02-26 22:38:34 +00002548 AddressingModeMatcher(SmallVectorImpl<Instruction *> &AMI,
Igor Laevsky3be81ba2017-02-07 13:27:20 +00002549 const TargetLowering &TLI,
2550 const TargetRegisterInfo &TRI,
2551 Type *AT, unsigned AS,
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00002552 Instruction *MI, ExtAddrMode &AM,
Ahmed Bougachaf3299142015-06-17 20:44:32 +00002553 const SetOfInstrs &InsertedInsts,
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002554 InstrToOrigTy &PromotedInsts,
2555 TypePromotionTransaction &TPT)
Igor Laevsky3be81ba2017-02-07 13:27:20 +00002556 : AddrModeInsts(AMI), TLI(TLI), TRI(TRI),
Mehdi Amini4fe37982015-07-07 18:45:17 +00002557 DL(MI->getModule()->getDataLayout()), AccessTy(AT), AddrSpace(AS),
2558 MemoryInst(MI), AddrMode(AM), InsertedInsts(InsertedInsts),
2559 PromotedInsts(PromotedInsts), TPT(TPT) {
Chandler Carruthc8925912013-01-05 02:09:22 +00002560 IgnoreProfitability = false;
2561 }
Stephen Lin837bba12013-07-15 17:55:02 +00002562
Eugene Zelenko900b6332017-08-29 22:32:07 +00002563public:
Sanjay Patel4ac6b112015-09-21 22:47:23 +00002564 /// Find the maximal addressing mode that a load/store of V can fold,
Chandler Carruthc8925912013-01-05 02:09:22 +00002565 /// give an access type of AccessTy. This returns a list of involved
2566 /// instructions in AddrModeInsts.
Ahmed Bougachaf3299142015-06-17 20:44:32 +00002567 /// \p InsertedInsts The instructions inserted by other CodeGenPrepare
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002568 /// optimizations.
2569 /// \p PromotedInsts maps the instructions to their type before promotion.
2570 /// \p The ongoing transaction where every action should be registered.
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00002571 static ExtAddrMode Match(Value *V, Type *AccessTy, unsigned AS,
Chandler Carruthc8925912013-01-05 02:09:22 +00002572 Instruction *MemoryInst,
2573 SmallVectorImpl<Instruction*> &AddrModeInsts,
Igor Laevsky3be81ba2017-02-07 13:27:20 +00002574 const TargetLowering &TLI,
2575 const TargetRegisterInfo &TRI,
Ahmed Bougachaf3299142015-06-17 20:44:32 +00002576 const SetOfInstrs &InsertedInsts,
Quentin Colombet3a4bf042014-02-06 21:44:56 +00002577 InstrToOrigTy &PromotedInsts,
2578 TypePromotionTransaction &TPT) {
Chandler Carruthc8925912013-01-05 02:09:22 +00002579 ExtAddrMode Result;
2580
Igor Laevsky3be81ba2017-02-07 13:27:20 +00002581 bool Success = AddressingModeMatcher(AddrModeInsts, TLI, TRI,
2582 AccessTy, AS,
Ahmed Bougachaf3299142015-06-17 20:44:32 +00002583 MemoryInst, Result, InsertedInsts,
Sanjay Patelfc580a62015-09-21 23:03:16 +00002584 PromotedInsts, TPT).matchAddr(V, 0);
Chandler Carruthc8925912013-01-05 02:09:22 +00002585 (void)Success; assert(Success && "Couldn't select *anything*?");
2586 return Result;
2587 }
Eugene Zelenko900b6332017-08-29 22:32:07 +00002588
Chandler Carruthc8925912013-01-05 02:09:22 +00002589private:
Sanjay Patelfc580a62015-09-21 23:03:16 +00002590 bool matchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth);
2591 bool matchAddr(Value *V, unsigned Depth);
2592 bool matchOperationAddr(User *Operation, unsigned Opcode, unsigned Depth,
Craig Topperc0196b12014-04-14 00:51:57 +00002593 bool *MovedAway = nullptr);
Sanjay Patelfc580a62015-09-21 23:03:16 +00002594 bool isProfitableToFoldIntoAddressingMode(Instruction *I,
Chandler Carruthc8925912013-01-05 02:09:22 +00002595 ExtAddrMode &AMBefore,
2596 ExtAddrMode &AMAfter);
Sanjay Patelfc580a62015-09-21 23:03:16 +00002597 bool valueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2);
2598 bool isPromotionProfitable(unsigned NewCost, unsigned OldCost,
Quentin Colombet867c5502014-02-14 22:23:22 +00002599 Value *PromotedOperand) const;
Chandler Carruthc8925912013-01-05 02:09:22 +00002600};
2601
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002602/// \brief Keep track of simplification of Phi nodes.
2603/// Accept the set of all phi nodes and erase phi node from this set
2604/// if it is simplified.
2605class SimplificationTracker {
2606 DenseMap<Value *, Value *> Storage;
2607 const SimplifyQuery &SQ;
2608 SmallPtrSetImpl<PHINode *> &AllPhiNodes;
2609 SmallPtrSetImpl<SelectInst *> &AllSelectNodes;
2610
2611public:
2612 SimplificationTracker(const SimplifyQuery &sq,
2613 SmallPtrSetImpl<PHINode *> &APN,
2614 SmallPtrSetImpl<SelectInst *> &ASN)
2615 : SQ(sq), AllPhiNodes(APN), AllSelectNodes(ASN) {}
2616
2617 Value *Get(Value *V) {
2618 do {
2619 auto SV = Storage.find(V);
2620 if (SV == Storage.end())
2621 return V;
2622 V = SV->second;
2623 } while (true);
2624 }
2625
2626 Value *Simplify(Value *Val) {
2627 SmallVector<Value *, 32> WorkList;
2628 SmallPtrSet<Value *, 32> Visited;
2629 WorkList.push_back(Val);
2630 while (!WorkList.empty()) {
2631 auto P = WorkList.pop_back_val();
2632 if (!Visited.insert(P).second)
2633 continue;
2634 if (auto *PI = dyn_cast<Instruction>(P))
2635 if (Value *V = SimplifyInstruction(cast<Instruction>(PI), SQ)) {
2636 for (auto *U : PI->users())
2637 WorkList.push_back(cast<Value>(U));
2638 Put(PI, V);
2639 PI->replaceAllUsesWith(V);
2640 if (auto *PHI = dyn_cast<PHINode>(PI))
2641 AllPhiNodes.erase(PHI);
2642 if (auto *Select = dyn_cast<SelectInst>(PI))
2643 AllSelectNodes.erase(Select);
2644 PI->eraseFromParent();
2645 }
2646 }
2647 return Get(Val);
2648 }
2649
2650 void Put(Value *From, Value *To) {
2651 Storage.insert({ From, To });
2652 }
2653};
2654
John Brawn736bf002017-10-03 13:08:22 +00002655/// \brief A helper class for combining addressing modes.
2656class AddressingModeCombiner {
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002657 typedef std::pair<Value *, BasicBlock *> ValueInBB;
2658 typedef DenseMap<ValueInBB, Value *> FoldAddrToValueMapping;
2659 typedef std::pair<PHINode *, PHINode *> PHIPair;
2660
John Brawn736bf002017-10-03 13:08:22 +00002661private:
2662 /// The addressing modes we've collected.
2663 SmallVector<ExtAddrMode, 16> AddrModes;
2664
2665 /// The field in which the AddrModes differ, when we have more than one.
2666 ExtAddrMode::FieldName DifferentField = ExtAddrMode::NoField;
2667
2668 /// Are the AddrModes that we have all just equal to their original values?
2669 bool AllAddrModesTrivial = true;
2670
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002671 /// Common Type for all different fields in addressing modes.
2672 Type *CommonType;
2673
2674 /// SimplifyQuery for simplifyInstruction utility.
2675 const SimplifyQuery &SQ;
2676
2677 /// Original Address.
2678 ValueInBB Original;
2679
John Brawn736bf002017-10-03 13:08:22 +00002680public:
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002681 AddressingModeCombiner(const SimplifyQuery &_SQ, ValueInBB OriginalValue)
2682 : CommonType(nullptr), SQ(_SQ), Original(OriginalValue) {}
2683
John Brawn736bf002017-10-03 13:08:22 +00002684 /// \brief Get the combined AddrMode
2685 const ExtAddrMode &getAddrMode() const {
2686 return AddrModes[0];
2687 }
2688
2689 /// \brief Add a new AddrMode if it's compatible with the AddrModes we already
2690 /// have.
2691 /// \return True iff we succeeded in doing so.
2692 bool addNewAddrMode(ExtAddrMode &NewAddrMode) {
2693 // Take note of if we have any non-trivial AddrModes, as we need to detect
2694 // when all AddrModes are trivial as then we would introduce a phi or select
2695 // which just duplicates what's already there.
2696 AllAddrModesTrivial = AllAddrModesTrivial && NewAddrMode.isTrivial();
2697
2698 // If this is the first addrmode then everything is fine.
2699 if (AddrModes.empty()) {
2700 AddrModes.emplace_back(NewAddrMode);
2701 return true;
2702 }
2703
2704 // Figure out how different this is from the other address modes, which we
2705 // can do just by comparing against the first one given that we only care
2706 // about the cumulative difference.
2707 ExtAddrMode::FieldName ThisDifferentField =
2708 AddrModes[0].compare(NewAddrMode);
2709 if (DifferentField == ExtAddrMode::NoField)
2710 DifferentField = ThisDifferentField;
2711 else if (DifferentField != ThisDifferentField)
2712 DifferentField = ExtAddrMode::MultipleFields;
2713
John Brawn70cdb5b2017-11-24 14:10:45 +00002714 // If NewAddrMode differs in only one dimension, and that dimension isn't
2715 // the amount that ScaledReg is scaled by, then we can handle it by
Serguei Katkov505359f2017-11-20 05:42:36 +00002716 // inserting a phi/select later on. Even if NewAddMode is the same
2717 // we still need to collect it due to original value is different.
2718 // And later we will need all original values as anchors during
2719 // finding the common Phi node.
John Brawn70cdb5b2017-11-24 14:10:45 +00002720 if (DifferentField != ExtAddrMode::MultipleFields &&
2721 DifferentField != ExtAddrMode::ScaleField) {
John Brawn736bf002017-10-03 13:08:22 +00002722 AddrModes.emplace_back(NewAddrMode);
2723 return true;
2724 }
2725
2726 // We couldn't combine NewAddrMode with the rest, so return failure.
2727 AddrModes.clear();
2728 return false;
2729 }
2730
2731 /// \brief Combine the addressing modes we've collected into a single
2732 /// addressing mode.
2733 /// \return True iff we successfully combined them or we only had one so
2734 /// didn't need to combine them anyway.
2735 bool combineAddrModes() {
2736 // If we have no AddrModes then they can't be combined.
2737 if (AddrModes.size() == 0)
2738 return false;
2739
2740 // A single AddrMode can trivially be combined.
Serguei Katkov505359f2017-11-20 05:42:36 +00002741 if (AddrModes.size() == 1 || DifferentField == ExtAddrMode::NoField)
John Brawn736bf002017-10-03 13:08:22 +00002742 return true;
2743
2744 // If the AddrModes we collected are all just equal to the value they are
2745 // derived from then combining them wouldn't do anything useful.
2746 if (AllAddrModesTrivial)
2747 return false;
2748
John Brawn70cdb5b2017-11-24 14:10:45 +00002749 if (!addrModeCombiningAllowed())
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002750 return false;
2751
2752 // Build a map between <original value, basic block where we saw it> to
2753 // value of base register.
Serguei Katkov50364592017-11-29 05:51:26 +00002754 // Bail out if there is no common type.
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002755 FoldAddrToValueMapping Map;
Serguei Katkov50364592017-11-29 05:51:26 +00002756 if (!initializeMap(Map))
2757 return false;
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002758
2759 Value *CommonValue = findCommon(Map);
2760 if (CommonValue)
John Brawn70cdb5b2017-11-24 14:10:45 +00002761 AddrModes[0].SetCombinedField(DifferentField, CommonValue, AddrModes);
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002762 return CommonValue != nullptr;
2763 }
2764
2765private:
2766 /// \brief Initialize Map with anchor values. For address seen in some BB
2767 /// we set the value of different field saw in this address.
2768 /// If address is not an instruction than basic block is set to null.
2769 /// At the same time we find a common type for different field we will
2770 /// use to create new Phi/Select nodes. Keep it in CommonType field.
Serguei Katkov50364592017-11-29 05:51:26 +00002771 /// Return false if there is no common type found.
2772 bool initializeMap(FoldAddrToValueMapping &Map) {
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002773 // Keep track of keys where the value is null. We will need to replace it
2774 // with constant null when we know the common type.
2775 SmallVector<ValueInBB, 2> NullValue;
John Brawn70cdb5b2017-11-24 14:10:45 +00002776 Type *IntPtrTy = SQ.DL.getIntPtrType(AddrModes[0].OriginalValue->getType());
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002777 for (auto &AM : AddrModes) {
2778 BasicBlock *BB = nullptr;
2779 if (Instruction *I = dyn_cast<Instruction>(AM.OriginalValue))
2780 BB = I->getParent();
2781
John Brawn70cdb5b2017-11-24 14:10:45 +00002782 Value *DV = AM.GetFieldAsValue(DifferentField, IntPtrTy);
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002783 if (DV) {
Serguei Katkov50364592017-11-29 05:51:26 +00002784 auto *Type = DV->getType();
2785 if (CommonType && CommonType != Type)
2786 return false;
2787 CommonType = Type;
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002788 Map[{ AM.OriginalValue, BB }] = DV;
2789 } else {
2790 NullValue.push_back({ AM.OriginalValue, BB });
2791 }
2792 }
2793 assert(CommonType && "At least one non-null value must be!");
2794 for (auto VIBB : NullValue)
2795 Map[VIBB] = Constant::getNullValue(CommonType);
Serguei Katkov50364592017-11-29 05:51:26 +00002796 return true;
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002797 }
2798
2799 /// \brief We have mapping between value A and basic block where value A
2800 /// seen to other value B where B was a field in addressing mode represented
2801 /// by A. Also we have an original value C representin an address in some
2802 /// basic block. Traversing from C through phi and selects we ended up with
2803 /// A's in a map. This utility function tries to find a value V which is a
2804 /// field in addressing mode C and traversing through phi nodes and selects
2805 /// we will end up in corresponded values B in a map.
2806 /// The utility will create a new Phi/Selects if needed.
2807 // The simple example looks as follows:
2808 // BB1:
2809 // p1 = b1 + 40
2810 // br cond BB2, BB3
2811 // BB2:
2812 // p2 = b2 + 40
2813 // br BB3
2814 // BB3:
2815 // p = phi [p1, BB1], [p2, BB2]
2816 // v = load p
2817 // Map is
2818 // <p1, BB1> -> b1
2819 // <p2, BB2> -> b2
2820 // Request is
2821 // <p, BB3> -> ?
2822 // The function tries to find or build phi [b1, BB1], [b2, BB2] in BB3
2823 Value *findCommon(FoldAddrToValueMapping &Map) {
2824 // Tracks of new created Phi nodes.
2825 SmallPtrSet<PHINode *, 32> NewPhiNodes;
2826 // Tracks of new created Select nodes.
2827 SmallPtrSet<SelectInst *, 32> NewSelectNodes;
2828 // Tracks the simplification of new created phi nodes. The reason we use
2829 // this mapping is because we will add new created Phi nodes in AddrToBase.
2830 // Simplification of Phi nodes is recursive, so some Phi node may
2831 // be simplified after we added it to AddrToBase.
2832 // Using this mapping we can find the current value in AddrToBase.
2833 SimplificationTracker ST(SQ, NewPhiNodes, NewSelectNodes);
2834
2835 // First step, DFS to create PHI nodes for all intermediate blocks.
2836 // Also fill traverse order for the second step.
2837 SmallVector<ValueInBB, 32> TraverseOrder;
2838 InsertPlaceholders(Map, TraverseOrder, NewPhiNodes, NewSelectNodes);
2839
2840 // Second Step, fill new nodes by merged values and simplify if possible.
2841 FillPlaceholders(Map, TraverseOrder, ST);
2842
2843 if (!AddrSinkNewSelects && NewSelectNodes.size() > 0) {
2844 DestroyNodes(NewPhiNodes);
2845 DestroyNodes(NewSelectNodes);
2846 return nullptr;
2847 }
2848
2849 // Now we'd like to match New Phi nodes to existed ones.
2850 unsigned PhiNotMatchedCount = 0;
2851 if (!MatchPhiSet(NewPhiNodes, ST, AddrSinkNewPhis, PhiNotMatchedCount)) {
2852 DestroyNodes(NewPhiNodes);
2853 DestroyNodes(NewSelectNodes);
2854 return nullptr;
2855 }
2856
2857 auto *Result = ST.Get(Map.find(Original)->second);
2858 if (Result) {
2859 NumMemoryInstsPhiCreated += NewPhiNodes.size() + PhiNotMatchedCount;
2860 NumMemoryInstsSelectCreated += NewSelectNodes.size();
2861 }
2862 return Result;
2863 }
2864
2865 /// \brief Destroy nodes from a set.
2866 template <typename T> void DestroyNodes(SmallPtrSetImpl<T *> &Instructions) {
2867 // For safe erasing, replace the Phi with dummy value first.
2868 auto Dummy = UndefValue::get(CommonType);
2869 for (auto I : Instructions) {
2870 I->replaceAllUsesWith(Dummy);
2871 I->eraseFromParent();
2872 }
2873 }
2874
2875 /// \brief Try to match PHI node to Candidate.
2876 /// Matcher tracks the matched Phi nodes.
2877 bool MatchPhiNode(PHINode *PHI, PHINode *Candidate,
2878 DenseSet<PHIPair> &Matcher,
2879 SmallPtrSetImpl<PHINode *> &PhiNodesToMatch) {
2880 SmallVector<PHIPair, 8> WorkList;
2881 Matcher.insert({ PHI, Candidate });
2882 WorkList.push_back({ PHI, Candidate });
2883 SmallSet<PHIPair, 8> Visited;
2884 while (!WorkList.empty()) {
2885 auto Item = WorkList.pop_back_val();
2886 if (!Visited.insert(Item).second)
2887 continue;
2888 // We iterate over all incoming values to Phi to compare them.
2889 // If values are different and both of them Phi and the first one is a
2890 // Phi we added (subject to match) and both of them is in the same basic
2891 // block then we can match our pair if values match. So we state that
2892 // these values match and add it to work list to verify that.
2893 for (auto B : Item.first->blocks()) {
2894 Value *FirstValue = Item.first->getIncomingValueForBlock(B);
2895 Value *SecondValue = Item.second->getIncomingValueForBlock(B);
2896 if (FirstValue == SecondValue)
2897 continue;
2898
2899 PHINode *FirstPhi = dyn_cast<PHINode>(FirstValue);
2900 PHINode *SecondPhi = dyn_cast<PHINode>(SecondValue);
2901
2902 // One of them is not Phi or
2903 // The first one is not Phi node from the set we'd like to match or
2904 // Phi nodes from different basic blocks then
2905 // we will not be able to match.
2906 if (!FirstPhi || !SecondPhi || !PhiNodesToMatch.count(FirstPhi) ||
2907 FirstPhi->getParent() != SecondPhi->getParent())
2908 return false;
2909
2910 // If we already matched them then continue.
2911 if (Matcher.count({ FirstPhi, SecondPhi }))
2912 continue;
2913 // So the values are different and does not match. So we need them to
2914 // match.
2915 Matcher.insert({ FirstPhi, SecondPhi });
2916 // But me must check it.
2917 WorkList.push_back({ FirstPhi, SecondPhi });
2918 }
2919 }
2920 return true;
2921 }
2922
2923 /// \brief For the given set of PHI nodes try to find their equivalents.
2924 /// Returns false if this matching fails and creation of new Phi is disabled.
2925 bool MatchPhiSet(SmallPtrSetImpl<PHINode *> &PhiNodesToMatch,
2926 SimplificationTracker &ST, bool AllowNewPhiNodes,
2927 unsigned &PhiNotMatchedCount) {
2928 DenseSet<PHIPair> Matched;
2929 SmallPtrSet<PHINode *, 8> WillNotMatch;
2930 while (PhiNodesToMatch.size()) {
2931 PHINode *PHI = *PhiNodesToMatch.begin();
2932
2933 // Add us, if no Phi nodes in the basic block we do not match.
2934 WillNotMatch.clear();
2935 WillNotMatch.insert(PHI);
2936
2937 // Traverse all Phis until we found equivalent or fail to do that.
2938 bool IsMatched = false;
2939 for (auto &P : PHI->getParent()->phis()) {
2940 if (&P == PHI)
2941 continue;
2942 if ((IsMatched = MatchPhiNode(PHI, &P, Matched, PhiNodesToMatch)))
2943 break;
2944 // If it does not match, collect all Phi nodes from matcher.
2945 // if we end up with no match, them all these Phi nodes will not match
2946 // later.
2947 for (auto M : Matched)
2948 WillNotMatch.insert(M.first);
2949 Matched.clear();
2950 }
2951 if (IsMatched) {
2952 // Replace all matched values and erase them.
2953 for (auto MV : Matched) {
2954 MV.first->replaceAllUsesWith(MV.second);
2955 PhiNodesToMatch.erase(MV.first);
2956 ST.Put(MV.first, MV.second);
2957 MV.first->eraseFromParent();
2958 }
2959 Matched.clear();
2960 continue;
2961 }
2962 // If we are not allowed to create new nodes then bail out.
2963 if (!AllowNewPhiNodes)
2964 return false;
2965 // Just remove all seen values in matcher. They will not match anything.
2966 PhiNotMatchedCount += WillNotMatch.size();
2967 for (auto *P : WillNotMatch)
2968 PhiNodesToMatch.erase(P);
2969 }
2970 return true;
2971 }
2972 /// \brief Fill the placeholder with values from predecessors and simplify it.
2973 void FillPlaceholders(FoldAddrToValueMapping &Map,
2974 SmallVectorImpl<ValueInBB> &TraverseOrder,
2975 SimplificationTracker &ST) {
2976 while (!TraverseOrder.empty()) {
2977 auto Current = TraverseOrder.pop_back_val();
2978 assert(Map.find(Current) != Map.end() && "No node to fill!!!");
2979 Value *CurrentValue = Current.first;
2980 BasicBlock *CurrentBlock = Current.second;
2981 Value *V = Map[Current];
2982
2983 if (SelectInst *Select = dyn_cast<SelectInst>(V)) {
2984 // CurrentValue also must be Select.
2985 auto *CurrentSelect = cast<SelectInst>(CurrentValue);
2986 auto *TrueValue = CurrentSelect->getTrueValue();
2987 ValueInBB TrueItem = { TrueValue, isa<Instruction>(TrueValue)
2988 ? CurrentBlock
2989 : nullptr };
2990 assert(Map.find(TrueItem) != Map.end() && "No True Value!");
Serguei Katkovb0b67a82017-12-18 04:25:07 +00002991 Select->setTrueValue(ST.Get(Map[TrueItem]));
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002992 auto *FalseValue = CurrentSelect->getFalseValue();
2993 ValueInBB FalseItem = { FalseValue, isa<Instruction>(FalseValue)
2994 ? CurrentBlock
2995 : nullptr };
2996 assert(Map.find(FalseItem) != Map.end() && "No False Value!");
Serguei Katkovb0b67a82017-12-18 04:25:07 +00002997 Select->setFalseValue(ST.Get(Map[FalseItem]));
Serguei Katkovd5d8d542017-11-05 05:50:33 +00002998 } else {
2999 // Must be a Phi node then.
3000 PHINode *PHI = cast<PHINode>(V);
3001 // Fill the Phi node with values from predecessors.
3002 bool IsDefinedInThisBB =
3003 cast<Instruction>(CurrentValue)->getParent() == CurrentBlock;
3004 auto *CurrentPhi = dyn_cast<PHINode>(CurrentValue);
3005 for (auto B : predecessors(CurrentBlock)) {
3006 Value *PV = IsDefinedInThisBB
3007 ? CurrentPhi->getIncomingValueForBlock(B)
3008 : CurrentValue;
3009 ValueInBB item = { PV, isa<Instruction>(PV) ? B : nullptr };
3010 assert(Map.find(item) != Map.end() && "No predecessor Value!");
3011 PHI->addIncoming(ST.Get(Map[item]), B);
3012 }
3013 }
3014 // Simplify if possible.
3015 Map[Current] = ST.Simplify(V);
3016 }
3017 }
3018
3019 /// Starting from value recursively iterates over predecessors up to known
3020 /// ending values represented in a map. For each traversed block inserts
3021 /// a placeholder Phi or Select.
3022 /// Reports all new created Phi/Select nodes by adding them to set.
3023 /// Also reports and order in what basic blocks have been traversed.
3024 void InsertPlaceholders(FoldAddrToValueMapping &Map,
3025 SmallVectorImpl<ValueInBB> &TraverseOrder,
3026 SmallPtrSetImpl<PHINode *> &NewPhiNodes,
3027 SmallPtrSetImpl<SelectInst *> &NewSelectNodes) {
3028 SmallVector<ValueInBB, 32> Worklist;
3029 assert((isa<PHINode>(Original.first) || isa<SelectInst>(Original.first)) &&
3030 "Address must be a Phi or Select node");
3031 auto *Dummy = UndefValue::get(CommonType);
3032 Worklist.push_back(Original);
3033 while (!Worklist.empty()) {
3034 auto Current = Worklist.pop_back_val();
3035 // If value is not an instruction it is something global, constant,
3036 // parameter and we can say that this value is observable in any block.
3037 // Set block to null to denote it.
3038 // Also please take into account that it is how we build anchors.
3039 if (!isa<Instruction>(Current.first))
3040 Current.second = nullptr;
3041 // if it is already visited or it is an ending value then skip it.
3042 if (Map.find(Current) != Map.end())
3043 continue;
3044 TraverseOrder.push_back(Current);
3045
3046 Value *CurrentValue = Current.first;
3047 BasicBlock *CurrentBlock = Current.second;
3048 // CurrentValue must be a Phi node or select. All others must be covered
3049 // by anchors.
3050 Instruction *CurrentI = cast<Instruction>(CurrentValue);
3051 bool IsDefinedInThisBB = CurrentI->getParent() == CurrentBlock;
3052
3053 unsigned PredCount =
3054 std::distance(pred_begin(CurrentBlock), pred_end(CurrentBlock));
3055 // if Current Value is not defined in this basic block we are interested
3056 // in values in predecessors.
3057 if (!IsDefinedInThisBB) {
3058 assert(PredCount && "Unreachable block?!");
3059 PHINode *PHI = PHINode::Create(CommonType, PredCount, "sunk_phi",
3060 &CurrentBlock->front());
3061 Map[Current] = PHI;
3062 NewPhiNodes.insert(PHI);
3063 // Add all predecessors in work list.
3064 for (auto B : predecessors(CurrentBlock))
3065 Worklist.push_back({ CurrentValue, B });
3066 continue;
3067 }
3068 // Value is defined in this basic block.
3069 if (SelectInst *OrigSelect = dyn_cast<SelectInst>(CurrentI)) {
3070 // Is it OK to get metadata from OrigSelect?!
3071 // Create a Select placeholder with dummy value.
3072 SelectInst *Select =
3073 SelectInst::Create(OrigSelect->getCondition(), Dummy, Dummy,
3074 OrigSelect->getName(), OrigSelect, OrigSelect);
3075 Map[Current] = Select;
3076 NewSelectNodes.insert(Select);
3077 // We are interested in True and False value in this basic block.
3078 Worklist.push_back({ OrigSelect->getTrueValue(), CurrentBlock });
3079 Worklist.push_back({ OrigSelect->getFalseValue(), CurrentBlock });
3080 } else {
3081 // It must be a Phi node then.
3082 auto *CurrentPhi = cast<PHINode>(CurrentI);
3083 // Create new Phi node for merge of bases.
3084 assert(PredCount && "Unreachable block?!");
3085 PHINode *PHI = PHINode::Create(CommonType, PredCount, "sunk_phi",
3086 &CurrentBlock->front());
3087 Map[Current] = PHI;
3088 NewPhiNodes.insert(PHI);
3089
3090 // Add all predecessors in work list.
3091 for (auto B : predecessors(CurrentBlock))
3092 Worklist.push_back({ CurrentPhi->getIncomingValueForBlock(B), B });
3093 }
3094 }
John Brawn736bf002017-10-03 13:08:22 +00003095 }
John Brawn70cdb5b2017-11-24 14:10:45 +00003096
3097 bool addrModeCombiningAllowed() {
3098 if (DisableComplexAddrModes)
3099 return false;
3100 switch (DifferentField) {
3101 default:
3102 return false;
3103 case ExtAddrMode::BaseRegField:
3104 return AddrSinkCombineBaseReg;
3105 case ExtAddrMode::BaseGVField:
3106 return AddrSinkCombineBaseGV;
3107 case ExtAddrMode::BaseOffsField:
3108 return AddrSinkCombineBaseOffs;
3109 case ExtAddrMode::ScaledRegField:
3110 return AddrSinkCombineScaledReg;
3111 }
3112 }
John Brawn736bf002017-10-03 13:08:22 +00003113};
Eugene Zelenko900b6332017-08-29 22:32:07 +00003114} // end anonymous namespace
3115
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003116/// Try adding ScaleReg*Scale to the current addressing mode.
Chandler Carruthc8925912013-01-05 02:09:22 +00003117/// Return true and update AddrMode if this addr mode is legal for the target,
3118/// false if not.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003119bool AddressingModeMatcher::matchScaledValue(Value *ScaleReg, int64_t Scale,
Chandler Carruthc8925912013-01-05 02:09:22 +00003120 unsigned Depth) {
3121 // If Scale is 1, then this is the same as adding ScaleReg to the addressing
3122 // mode. Just process that directly.
3123 if (Scale == 1)
Sanjay Patelfc580a62015-09-21 23:03:16 +00003124 return matchAddr(ScaleReg, Depth);
Stephen Lin837bba12013-07-15 17:55:02 +00003125
Chandler Carruthc8925912013-01-05 02:09:22 +00003126 // If the scale is 0, it takes nothing to add this.
3127 if (Scale == 0)
3128 return true;
Stephen Lin837bba12013-07-15 17:55:02 +00003129
Chandler Carruthc8925912013-01-05 02:09:22 +00003130 // If we already have a scale of this value, we can add to it, otherwise, we
3131 // need an available scale field.
3132 if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg)
3133 return false;
3134
3135 ExtAddrMode TestAddrMode = AddrMode;
3136
3137 // Add scale to turn X*4+X*3 -> X*7. This could also do things like
3138 // [A+B + A*7] -> [B+A*8].
3139 TestAddrMode.Scale += Scale;
3140 TestAddrMode.ScaledReg = ScaleReg;
3141
3142 // If the new address isn't legal, bail out.
Mehdi Amini0cdec1e2015-07-09 02:09:40 +00003143 if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace))
Chandler Carruthc8925912013-01-05 02:09:22 +00003144 return false;
3145
3146 // It was legal, so commit it.
3147 AddrMode = TestAddrMode;
Stephen Lin837bba12013-07-15 17:55:02 +00003148
Chandler Carruthc8925912013-01-05 02:09:22 +00003149 // Okay, we decided that we can add ScaleReg+Scale to AddrMode. Check now
3150 // to see if ScaleReg is actually X+C. If so, we can turn this into adding
3151 // X*Scale + C*Scale to addr mode.
Craig Topperc0196b12014-04-14 00:51:57 +00003152 ConstantInt *CI = nullptr; Value *AddLHS = nullptr;
Chandler Carruthc8925912013-01-05 02:09:22 +00003153 if (isa<Instruction>(ScaleReg) && // not a constant expr.
3154 match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI)))) {
3155 TestAddrMode.ScaledReg = AddLHS;
3156 TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale;
Stephen Lin837bba12013-07-15 17:55:02 +00003157
Chandler Carruthc8925912013-01-05 02:09:22 +00003158 // If this addressing mode is legal, commit it and remember that we folded
3159 // this instruction.
Mehdi Amini0cdec1e2015-07-09 02:09:40 +00003160 if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) {
Chandler Carruthc8925912013-01-05 02:09:22 +00003161 AddrModeInsts.push_back(cast<Instruction>(ScaleReg));
3162 AddrMode = TestAddrMode;
3163 return true;
3164 }
3165 }
3166
3167 // Otherwise, not (x+c)*scale, just return what we have.
3168 return true;
3169}
3170
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003171/// This is a little filter, which returns true if an addressing computation
3172/// involving I might be folded into a load/store accessing it.
3173/// This doesn't need to be perfect, but needs to accept at least
Chandler Carruthc8925912013-01-05 02:09:22 +00003174/// the set of instructions that MatchOperationAddr can.
3175static bool MightBeFoldableInst(Instruction *I) {
3176 switch (I->getOpcode()) {
3177 case Instruction::BitCast:
Eli Benderskyf13a0562014-05-22 00:02:52 +00003178 case Instruction::AddrSpaceCast:
Chandler Carruthc8925912013-01-05 02:09:22 +00003179 // Don't touch identity bitcasts.
3180 if (I->getType() == I->getOperand(0)->getType())
3181 return false;
3182 return I->getType()->isPointerTy() || I->getType()->isIntegerTy();
3183 case Instruction::PtrToInt:
3184 // PtrToInt is always a noop, as we know that the int type is pointer sized.
3185 return true;
3186 case Instruction::IntToPtr:
3187 // We know the input is intptr_t, so this is foldable.
3188 return true;
3189 case Instruction::Add:
3190 return true;
3191 case Instruction::Mul:
3192 case Instruction::Shl:
3193 // Can only handle X*C and X << C.
3194 return isa<ConstantInt>(I->getOperand(1));
3195 case Instruction::GetElementPtr:
3196 return true;
3197 default:
3198 return false;
3199 }
3200}
3201
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003202/// \brief Check whether or not \p Val is a legal instruction for \p TLI.
3203/// \note \p Val is assumed to be the product of some type promotion.
3204/// Therefore if \p Val has an undefined state in \p TLI, this is assumed
3205/// to be legal, as the non-promoted value would have had the same state.
Mehdi Amini44ede332015-07-09 02:09:04 +00003206static bool isPromotedInstructionLegal(const TargetLowering &TLI,
3207 const DataLayout &DL, Value *Val) {
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003208 Instruction *PromotedInst = dyn_cast<Instruction>(Val);
3209 if (!PromotedInst)
3210 return false;
3211 int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode());
3212 // If the ISDOpcode is undefined, it was undefined before the promotion.
3213 if (!ISDOpcode)
3214 return true;
3215 // Otherwise, check if the promoted instruction is legal or not.
3216 return TLI.isOperationLegalOrCustom(
Mehdi Amini44ede332015-07-09 02:09:04 +00003217 ISDOpcode, TLI.getValueType(DL, PromotedInst->getType()));
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003218}
3219
Eugene Zelenko900b6332017-08-29 22:32:07 +00003220namespace {
3221
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003222/// \brief Hepler class to perform type promotion.
3223class TypePromotionHelper {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003224 /// \brief Utility function to check whether or not a sign or zero extension
3225 /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by
3226 /// either using the operands of \p Inst or promoting \p Inst.
3227 /// The type of the extension is defined by \p IsSExt.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003228 /// In other words, check if:
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003229 /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003230 /// #1 Promotion applies:
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003231 /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...).
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003232 /// #2 Operand reuses:
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003233 /// ext opnd1 to ConsideredExtType.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003234 /// \p PromotedInsts maps the instructions to their type before promotion.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003235 static bool canGetThrough(const Instruction *Inst, Type *ConsideredExtType,
3236 const InstrToOrigTy &PromotedInsts, bool IsSExt);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003237
3238 /// \brief Utility function to determine if \p OpIdx should be promoted when
3239 /// promoting \p Inst.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003240 static bool shouldExtOperand(const Instruction *Inst, int OpIdx) {
Rafael Espindola84921b92015-10-24 23:11:13 +00003241 return !(isa<SelectInst>(Inst) && OpIdx == 0);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003242 }
3243
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003244 /// \brief Utility function to promote the operand of \p Ext when this
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003245 /// operand is a promotable trunc or sext or zext.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003246 /// \p PromotedInsts maps the instructions to their type before promotion.
Quentin Colombet1b274f92015-03-10 21:48:15 +00003247 /// \p CreatedInstsCost[out] contains the cost of all instructions
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003248 /// created to promote the operand of Ext.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003249 /// Newly added extensions are inserted in \p Exts.
3250 /// Newly added truncates are inserted in \p Truncs.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003251 /// Should never be called directly.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003252 /// \return The promoted value which is used instead of Ext.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003253 static Value *promoteOperandForTruncAndAnyExt(
3254 Instruction *Ext, TypePromotionTransaction &TPT,
Quentin Colombet1b274f92015-03-10 21:48:15 +00003255 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003256 SmallVectorImpl<Instruction *> *Exts,
Quentin Colombet1b274f92015-03-10 21:48:15 +00003257 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003258
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003259 /// \brief Utility function to promote the operand of \p Ext when this
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003260 /// operand is promotable and is not a supported trunc or sext.
3261 /// \p PromotedInsts maps the instructions to their type before promotion.
Quentin Colombet1b274f92015-03-10 21:48:15 +00003262 /// \p CreatedInstsCost[out] contains the cost of all the instructions
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003263 /// created to promote the operand of Ext.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003264 /// Newly added extensions are inserted in \p Exts.
3265 /// Newly added truncates are inserted in \p Truncs.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003266 /// Should never be called directly.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003267 /// \return The promoted value which is used instead of Ext.
Quentin Colombet1b274f92015-03-10 21:48:15 +00003268 static Value *promoteOperandForOther(Instruction *Ext,
3269 TypePromotionTransaction &TPT,
3270 InstrToOrigTy &PromotedInsts,
3271 unsigned &CreatedInstsCost,
3272 SmallVectorImpl<Instruction *> *Exts,
3273 SmallVectorImpl<Instruction *> *Truncs,
3274 const TargetLowering &TLI, bool IsSExt);
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003275
3276 /// \see promoteOperandForOther.
Quentin Colombet1b274f92015-03-10 21:48:15 +00003277 static Value *signExtendOperandForOther(
3278 Instruction *Ext, TypePromotionTransaction &TPT,
3279 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
3280 SmallVectorImpl<Instruction *> *Exts,
3281 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
3282 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
3283 Exts, Truncs, TLI, true);
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003284 }
3285
3286 /// \see promoteOperandForOther.
Quentin Colombet1b274f92015-03-10 21:48:15 +00003287 static Value *zeroExtendOperandForOther(
3288 Instruction *Ext, TypePromotionTransaction &TPT,
3289 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
3290 SmallVectorImpl<Instruction *> *Exts,
3291 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
3292 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
3293 Exts, Truncs, TLI, false);
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003294 }
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003295
3296public:
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003297 /// Type for the utility function that promotes the operand of Ext.
Eugene Zelenko900b6332017-08-29 22:32:07 +00003298 using Action = Value *(*)(Instruction *Ext, TypePromotionTransaction &TPT,
3299 InstrToOrigTy &PromotedInsts,
3300 unsigned &CreatedInstsCost,
3301 SmallVectorImpl<Instruction *> *Exts,
3302 SmallVectorImpl<Instruction *> *Truncs,
3303 const TargetLowering &TLI);
3304
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003305 /// \brief Given a sign/zero extend instruction \p Ext, return the approriate
3306 /// action to promote the operand of \p Ext instead of using Ext.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003307 /// \return NULL if no promotable action is possible with the current
3308 /// sign extension.
Ahmed Bougachaf3299142015-06-17 20:44:32 +00003309 /// \p InsertedInsts keeps track of all the instructions inserted by the
3310 /// other CodeGenPrepare optimizations. This information is important
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003311 /// because we do not want to promote these instructions as CodeGenPrepare
3312 /// will reinsert them later. Thus creating an infinite loop: create/remove.
3313 /// \p PromotedInsts maps the instructions to their type before promotion.
Ahmed Bougachaf3299142015-06-17 20:44:32 +00003314 static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedInsts,
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003315 const TargetLowering &TLI,
3316 const InstrToOrigTy &PromotedInsts);
3317};
3318
Eugene Zelenko900b6332017-08-29 22:32:07 +00003319} // end anonymous namespace
3320
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003321bool TypePromotionHelper::canGetThrough(const Instruction *Inst,
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003322 Type *ConsideredExtType,
3323 const InstrToOrigTy &PromotedInsts,
3324 bool IsSExt) {
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003325 // The promotion helper does not know how to deal with vector types yet.
3326 // To be able to fix that, we would need to fix the places where we
3327 // statically extend, e.g., constants and such.
3328 if (Inst->getType()->isVectorTy())
3329 return false;
3330
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003331 // We can always get through zext.
3332 if (isa<ZExtInst>(Inst))
3333 return true;
3334
3335 // sext(sext) is ok too.
3336 if (IsSExt && isa<SExtInst>(Inst))
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003337 return true;
3338
3339 // We can get through binary operator, if it is legal. In other words, the
3340 // binary operator must have a nuw or nsw flag.
3341 const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Inst);
3342 if (BinOp && isa<OverflowingBinaryOperator>(BinOp) &&
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003343 ((!IsSExt && BinOp->hasNoUnsignedWrap()) ||
3344 (IsSExt && BinOp->hasNoSignedWrap())))
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003345 return true;
3346
3347 // Check if we can do the following simplification.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003348 // ext(trunc(opnd)) --> ext(opnd)
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003349 if (!isa<TruncInst>(Inst))
3350 return false;
3351
3352 Value *OpndVal = Inst->getOperand(0);
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003353 // Check if we can use this operand in the extension.
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00003354 // If the type is larger than the result type of the extension, we cannot.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003355 if (!OpndVal->getType()->isIntegerTy() ||
3356 OpndVal->getType()->getIntegerBitWidth() >
3357 ConsideredExtType->getIntegerBitWidth())
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003358 return false;
3359
3360 // If the operand of the truncate is not an instruction, we will not have
3361 // any information on the dropped bits.
3362 // (Actually we could for constant but it is not worth the extra logic).
3363 Instruction *Opnd = dyn_cast<Instruction>(OpndVal);
3364 if (!Opnd)
3365 return false;
3366
3367 // Check if the source of the type is narrow enough.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003368 // I.e., check that trunc just drops extended bits of the same kind of
3369 // the extension.
3370 // #1 get the type of the operand and check the kind of the extended bits.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003371 const Type *OpndType;
3372 InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
Benjamin Kramer4cd5faa2015-07-31 17:00:39 +00003373 if (It != PromotedInsts.end() && It->second.getInt() == IsSExt)
3374 OpndType = It->second.getPointer();
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003375 else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))
3376 OpndType = Opnd->getOperand(0)->getType();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003377 else
3378 return false;
3379
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00003380 // #2 check that the truncate just drops extended bits.
Rafael Espindola84921b92015-10-24 23:11:13 +00003381 return Inst->getType()->getIntegerBitWidth() >=
3382 OpndType->getIntegerBitWidth();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003383}
3384
3385TypePromotionHelper::Action TypePromotionHelper::getAction(
Ahmed Bougachaf3299142015-06-17 20:44:32 +00003386 Instruction *Ext, const SetOfInstrs &InsertedInsts,
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003387 const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003388 assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
3389 "Unexpected instruction type");
3390 Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0));
3391 Type *ExtTy = Ext->getType();
3392 bool IsSExt = isa<SExtInst>(Ext);
3393 // If the operand of the extension is not an instruction, we cannot
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003394 // get through.
3395 // If it, check we can get through.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003396 if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))
Craig Topperc0196b12014-04-14 00:51:57 +00003397 return nullptr;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003398
3399 // Do not promote if the operand has been added by codegenprepare.
3400 // Otherwise, it means we are undoing an optimization that is likely to be
3401 // redone, thus causing potential infinite loop.
Ahmed Bougachaf3299142015-06-17 20:44:32 +00003402 if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))
Craig Topperc0196b12014-04-14 00:51:57 +00003403 return nullptr;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003404
3405 // SExt or Trunc instructions.
3406 // Return the related handler.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003407 if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) ||
3408 isa<ZExtInst>(ExtOpnd))
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003409 return promoteOperandForTruncAndAnyExt;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003410
3411 // Regular instruction.
3412 // Abort early if we will have to insert non-free instructions.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003413 if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType()))
Craig Topperc0196b12014-04-14 00:51:57 +00003414 return nullptr;
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003415 return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003416}
3417
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003418Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(
Eugene Zelenko900b6332017-08-29 22:32:07 +00003419 Instruction *SExt, TypePromotionTransaction &TPT,
Quentin Colombet1b274f92015-03-10 21:48:15 +00003420 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003421 SmallVectorImpl<Instruction *> *Exts,
Quentin Colombet1b274f92015-03-10 21:48:15 +00003422 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003423 // By construction, the operand of SExt is an instruction. Otherwise we cannot
3424 // get through it and this method should not be called.
3425 Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0));
Quentin Colombetac55b152014-09-16 22:36:07 +00003426 Value *ExtVal = SExt;
Quentin Colombet1b274f92015-03-10 21:48:15 +00003427 bool HasMergedNonFreeExt = false;
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003428 if (isa<ZExtInst>(SExtOpnd)) {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003429 // Replace s|zext(zext(opnd))
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003430 // => zext(opnd).
Quentin Colombet1b274f92015-03-10 21:48:15 +00003431 HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd);
Quentin Colombetac55b152014-09-16 22:36:07 +00003432 Value *ZExt =
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003433 TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType());
3434 TPT.replaceAllUsesWith(SExt, ZExt);
3435 TPT.eraseInstruction(SExt);
Quentin Colombetac55b152014-09-16 22:36:07 +00003436 ExtVal = ZExt;
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003437 } else {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003438 // Replace z|sext(trunc(opnd)) or sext(sext(opnd))
3439 // => z|sext(opnd).
Quentin Colombetb2c5c6d2014-09-11 21:22:14 +00003440 TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0));
3441 }
Quentin Colombet1b274f92015-03-10 21:48:15 +00003442 CreatedInstsCost = 0;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003443
3444 // Remove dead code.
3445 if (SExtOpnd->use_empty())
3446 TPT.eraseInstruction(SExtOpnd);
3447
Quentin Colombet9dcb7242014-09-15 18:26:58 +00003448 // Check if the extension is still needed.
Quentin Colombetac55b152014-09-16 22:36:07 +00003449 Instruction *ExtInst = dyn_cast<Instruction>(ExtVal);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003450 if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) {
Quentin Colombet1b274f92015-03-10 21:48:15 +00003451 if (ExtInst) {
3452 if (Exts)
3453 Exts->push_back(ExtInst);
3454 CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt;
3455 }
Quentin Colombetac55b152014-09-16 22:36:07 +00003456 return ExtVal;
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003457 }
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003458
Quentin Colombet9dcb7242014-09-15 18:26:58 +00003459 // At this point we have: ext ty opnd to ty.
3460 // Reassign the uses of ExtInst to the opnd and remove ExtInst.
3461 Value *NextVal = ExtInst->getOperand(0);
3462 TPT.eraseInstruction(ExtInst, NextVal);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003463 return NextVal;
3464}
3465
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003466Value *TypePromotionHelper::promoteOperandForOther(
3467 Instruction *Ext, TypePromotionTransaction &TPT,
Quentin Colombet1b274f92015-03-10 21:48:15 +00003468 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003469 SmallVectorImpl<Instruction *> *Exts,
Quentin Colombet1b274f92015-03-10 21:48:15 +00003470 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI,
3471 bool IsSExt) {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003472 // By construction, the operand of Ext is an instruction. Otherwise we cannot
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003473 // get through it and this method should not be called.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003474 Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0));
Quentin Colombet1b274f92015-03-10 21:48:15 +00003475 CreatedInstsCost = 0;
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003476 if (!ExtOpnd->hasOneUse()) {
3477 // ExtOpnd will be promoted.
3478 // All its uses, but Ext, will need to use a truncated value of the
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003479 // promoted version.
3480 // Create the truncate now.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003481 Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType());
Quentin Colombetac55b152014-09-16 22:36:07 +00003482 if (Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) {
Quentin Colombetac55b152014-09-16 22:36:07 +00003483 // Insert it just after the definition.
Sanjay Patel674d2c22017-08-29 14:07:48 +00003484 ITrunc->moveAfter(ExtOpnd);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003485 if (Truncs)
3486 Truncs->push_back(ITrunc);
Quentin Colombetac55b152014-09-16 22:36:07 +00003487 }
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003488
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003489 TPT.replaceAllUsesWith(ExtOpnd, Trunc);
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00003490 // Restore the operand of Ext (which has been replaced by the previous call
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003491 // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003492 TPT.setOperand(Ext, 0, ExtOpnd);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003493 }
3494
3495 // Get through the Instruction:
3496 // 1. Update its type.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003497 // 2. Replace the uses of Ext by Inst.
3498 // 3. Extend each operand that needs to be extended.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003499
3500 // Remember the original type of the instruction before promotion.
3501 // This is useful to know that the high bits are sign extended bits.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003502 PromotedInsts.insert(std::pair<Instruction *, TypeIsSExt>(
3503 ExtOpnd, TypeIsSExt(ExtOpnd->getType(), IsSExt)));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003504 // Step #1.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003505 TPT.mutateType(ExtOpnd, Ext->getType());
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003506 // Step #2.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003507 TPT.replaceAllUsesWith(Ext, ExtOpnd);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003508 // Step #3.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003509 Instruction *ExtForOpnd = Ext;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003510
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003511 DEBUG(dbgs() << "Propagate Ext to operands\n");
3512 for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003513 ++OpIdx) {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003514 DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n');
3515 if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() ||
3516 !shouldExtOperand(ExtOpnd, OpIdx)) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003517 DEBUG(dbgs() << "No need to propagate\n");
3518 continue;
3519 }
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003520 // Check if we can statically extend the operand.
3521 Value *Opnd = ExtOpnd->getOperand(OpIdx);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003522 if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003523 DEBUG(dbgs() << "Statically extend\n");
3524 unsigned BitWidth = Ext->getType()->getIntegerBitWidth();
3525 APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth)
3526 : Cst->getValue().zext(BitWidth);
3527 TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003528 continue;
3529 }
3530 // UndefValue are typed, so we have to statically sign extend them.
3531 if (isa<UndefValue>(Opnd)) {
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003532 DEBUG(dbgs() << "Statically extend\n");
3533 TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType()));
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003534 continue;
3535 }
3536
3537 // Otherwise we have to explicity sign extend the operand.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003538 // Check if Ext was reused to extend an operand.
3539 if (!ExtForOpnd) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003540 // If yes, create a new one.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003541 DEBUG(dbgs() << "More operands to ext\n");
Quentin Colombet84f89cc2014-12-22 18:11:52 +00003542 Value *ValForExtOpnd = IsSExt ? TPT.createSExt(Ext, Opnd, Ext->getType())
3543 : TPT.createZExt(Ext, Opnd, Ext->getType());
3544 if (!isa<Instruction>(ValForExtOpnd)) {
3545 TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);
3546 continue;
3547 }
3548 ExtForOpnd = cast<Instruction>(ValForExtOpnd);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003549 }
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003550 if (Exts)
3551 Exts->push_back(ExtForOpnd);
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003552 TPT.setOperand(ExtForOpnd, 0, Opnd);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003553
3554 // Move the sign extension before the insertion point.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003555 TPT.moveBefore(ExtForOpnd, ExtOpnd);
3556 TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd);
Quentin Colombet1b274f92015-03-10 21:48:15 +00003557 CreatedInstsCost += !TLI.isExtFree(ExtForOpnd);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003558 // If more sext are required, new instructions will have to be created.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003559 ExtForOpnd = nullptr;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003560 }
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003561 if (ExtForOpnd == Ext) {
3562 DEBUG(dbgs() << "Extension is useless now\n");
3563 TPT.eraseInstruction(Ext);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003564 }
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003565 return ExtOpnd;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003566}
3567
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003568/// Check whether or not promoting an instruction to a wider type is profitable.
Quentin Colombet1b274f92015-03-10 21:48:15 +00003569/// \p NewCost gives the cost of extension instructions created by the
3570/// promotion.
3571/// \p OldCost gives the cost of extension instructions before the promotion
3572/// plus the number of instructions that have been
3573/// matched in the addressing mode the promotion.
Quentin Colombet867c5502014-02-14 22:23:22 +00003574/// \p PromotedOperand is the value that has been promoted.
3575/// \return True if the promotion is profitable, false otherwise.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003576bool AddressingModeMatcher::isPromotionProfitable(
Quentin Colombet1b274f92015-03-10 21:48:15 +00003577 unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const {
3578 DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost << '\n');
3579 // The cost of the new extensions is greater than the cost of the
3580 // old extension plus what we folded.
Quentin Colombet867c5502014-02-14 22:23:22 +00003581 // This is not profitable.
Quentin Colombet1b274f92015-03-10 21:48:15 +00003582 if (NewCost > OldCost)
Quentin Colombet867c5502014-02-14 22:23:22 +00003583 return false;
Quentin Colombet1b274f92015-03-10 21:48:15 +00003584 if (NewCost < OldCost)
Quentin Colombet867c5502014-02-14 22:23:22 +00003585 return true;
3586 // The promotion is neutral but it may help folding the sign extension in
3587 // loads for instance.
3588 // Check that we did not create an illegal instruction.
Mehdi Amini44ede332015-07-09 02:09:04 +00003589 return isPromotedInstructionLegal(TLI, DL, PromotedOperand);
Quentin Colombet867c5502014-02-14 22:23:22 +00003590}
3591
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003592/// Given an instruction or constant expr, see if we can fold the operation
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00003593/// into the addressing mode. If so, update the addressing mode and return
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003594/// true, otherwise return false without modifying AddrMode.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003595/// If \p MovedAway is not NULL, it contains the information of whether or
3596/// not AddrInst has to be folded into the addressing mode on success.
3597/// If \p MovedAway == true, \p AddrInst will not be part of the addressing
3598/// because it has been moved away.
3599/// Thus AddrInst must not be added in the matched instructions.
3600/// This state can happen when AddrInst is a sext, since it may be moved away.
3601/// Therefore, AddrInst may not be valid when MovedAway is true and it must
3602/// not be referenced anymore.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003603bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003604 unsigned Depth,
3605 bool *MovedAway) {
Chandler Carruthc8925912013-01-05 02:09:22 +00003606 // Avoid exponential behavior on extremely deep expression trees.
3607 if (Depth >= 5) return false;
Stephen Lin837bba12013-07-15 17:55:02 +00003608
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003609 // By default, all matched instructions stay in place.
3610 if (MovedAway)
3611 *MovedAway = false;
3612
Chandler Carruthc8925912013-01-05 02:09:22 +00003613 switch (Opcode) {
3614 case Instruction::PtrToInt:
3615 // PtrToInt is always a noop, as we know that the int type is pointer sized.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003616 return matchAddr(AddrInst->getOperand(0), Depth);
Mehdi Amini44ede332015-07-09 02:09:04 +00003617 case Instruction::IntToPtr: {
3618 auto AS = AddrInst->getType()->getPointerAddressSpace();
3619 auto PtrTy = MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
Chandler Carruthc8925912013-01-05 02:09:22 +00003620 // This inttoptr is a no-op if the integer type is pointer sized.
Mehdi Amini44ede332015-07-09 02:09:04 +00003621 if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy)
Sanjay Patelfc580a62015-09-21 23:03:16 +00003622 return matchAddr(AddrInst->getOperand(0), Depth);
Chandler Carruthc8925912013-01-05 02:09:22 +00003623 return false;
Mehdi Amini44ede332015-07-09 02:09:04 +00003624 }
Chandler Carruthc8925912013-01-05 02:09:22 +00003625 case Instruction::BitCast:
3626 // BitCast is always a noop, and we can handle it as long as it is
3627 // int->int or pointer->pointer (we don't want int<->fp or something).
3628 if ((AddrInst->getOperand(0)->getType()->isPointerTy() ||
3629 AddrInst->getOperand(0)->getType()->isIntegerTy()) &&
3630 // Don't touch identity bitcasts. These were probably put here by LSR,
3631 // and we don't want to mess around with them. Assume it knows what it
3632 // is doing.
3633 AddrInst->getOperand(0)->getType() != AddrInst->getType())
Sanjay Patelfc580a62015-09-21 23:03:16 +00003634 return matchAddr(AddrInst->getOperand(0), Depth);
Chandler Carruthc8925912013-01-05 02:09:22 +00003635 return false;
Matt Arsenaultf05b0232015-05-26 16:59:43 +00003636 case Instruction::AddrSpaceCast: {
3637 unsigned SrcAS
3638 = AddrInst->getOperand(0)->getType()->getPointerAddressSpace();
3639 unsigned DestAS = AddrInst->getType()->getPointerAddressSpace();
3640 if (TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
Sanjay Patelfc580a62015-09-21 23:03:16 +00003641 return matchAddr(AddrInst->getOperand(0), Depth);
Matt Arsenaultf05b0232015-05-26 16:59:43 +00003642 return false;
3643 }
Chandler Carruthc8925912013-01-05 02:09:22 +00003644 case Instruction::Add: {
3645 // Check to see if we can merge in the RHS then the LHS. If so, we win.
3646 ExtAddrMode BackupAddrMode = AddrMode;
3647 unsigned OldSize = AddrModeInsts.size();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003648 // Start a transaction at this point.
3649 // The LHS may match but not the RHS.
3650 // Therefore, we need a higher level restoration point to undo partially
3651 // matched operation.
3652 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3653 TPT.getRestorationPoint();
3654
Sanjay Patelfc580a62015-09-21 23:03:16 +00003655 if (matchAddr(AddrInst->getOperand(1), Depth+1) &&
3656 matchAddr(AddrInst->getOperand(0), Depth+1))
Chandler Carruthc8925912013-01-05 02:09:22 +00003657 return true;
Stephen Lin837bba12013-07-15 17:55:02 +00003658
Chandler Carruthc8925912013-01-05 02:09:22 +00003659 // Restore the old addr mode info.
3660 AddrMode = BackupAddrMode;
3661 AddrModeInsts.resize(OldSize);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003662 TPT.rollback(LastKnownGood);
Stephen Lin837bba12013-07-15 17:55:02 +00003663
Chandler Carruthc8925912013-01-05 02:09:22 +00003664 // Otherwise this was over-aggressive. Try merging in the LHS then the RHS.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003665 if (matchAddr(AddrInst->getOperand(0), Depth+1) &&
3666 matchAddr(AddrInst->getOperand(1), Depth+1))
Chandler Carruthc8925912013-01-05 02:09:22 +00003667 return true;
Stephen Lin837bba12013-07-15 17:55:02 +00003668
Chandler Carruthc8925912013-01-05 02:09:22 +00003669 // Otherwise we definitely can't merge the ADD in.
3670 AddrMode = BackupAddrMode;
3671 AddrModeInsts.resize(OldSize);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003672 TPT.rollback(LastKnownGood);
Chandler Carruthc8925912013-01-05 02:09:22 +00003673 break;
3674 }
3675 //case Instruction::Or:
3676 // TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD.
3677 //break;
3678 case Instruction::Mul:
3679 case Instruction::Shl: {
3680 // Can only handle X*C and X << C.
3681 ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1));
Philip Reames9c3cbee2017-10-30 23:59:51 +00003682 if (!RHS || RHS->getBitWidth() > 64)
Sanjay Pateld3bbfa12014-07-16 22:40:28 +00003683 return false;
Chandler Carruthc8925912013-01-05 02:09:22 +00003684 int64_t Scale = RHS->getSExtValue();
3685 if (Opcode == Instruction::Shl)
3686 Scale = 1LL << Scale;
Stephen Lin837bba12013-07-15 17:55:02 +00003687
Sanjay Patelfc580a62015-09-21 23:03:16 +00003688 return matchScaledValue(AddrInst->getOperand(0), Scale, Depth);
Chandler Carruthc8925912013-01-05 02:09:22 +00003689 }
3690 case Instruction::GetElementPtr: {
3691 // Scan the GEP. We check it if it contains constant offsets and at most
3692 // one variable offset.
3693 int VariableOperand = -1;
3694 unsigned VariableScale = 0;
Stephen Lin837bba12013-07-15 17:55:02 +00003695
Chandler Carruthc8925912013-01-05 02:09:22 +00003696 int64_t ConstantOffset = 0;
Chandler Carruthc8925912013-01-05 02:09:22 +00003697 gep_type_iterator GTI = gep_type_begin(AddrInst);
3698 for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {
Peter Collingbourneab85225b2016-12-02 02:24:42 +00003699 if (StructType *STy = GTI.getStructTypeOrNull()) {
Mehdi Amini4fe37982015-07-07 18:45:17 +00003700 const StructLayout *SL = DL.getStructLayout(STy);
Chandler Carruthc8925912013-01-05 02:09:22 +00003701 unsigned Idx =
3702 cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();
3703 ConstantOffset += SL->getElementOffset(Idx);
3704 } else {
Mehdi Amini4fe37982015-07-07 18:45:17 +00003705 uint64_t TypeSize = DL.getTypeAllocSize(GTI.getIndexedType());
Chandler Carruthc8925912013-01-05 02:09:22 +00003706 if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {
3707 ConstantOffset += CI->getSExtValue()*TypeSize;
3708 } else if (TypeSize) { // Scales of zero don't do anything.
3709 // We only allow one variable index at the moment.
3710 if (VariableOperand != -1)
3711 return false;
Stephen Lin837bba12013-07-15 17:55:02 +00003712
Chandler Carruthc8925912013-01-05 02:09:22 +00003713 // Remember the variable index.
3714 VariableOperand = i;
3715 VariableScale = TypeSize;
3716 }
3717 }
3718 }
Stephen Lin837bba12013-07-15 17:55:02 +00003719
Chandler Carruthc8925912013-01-05 02:09:22 +00003720 // A common case is for the GEP to only do a constant offset. In this case,
3721 // just add it to the disp field and check validity.
3722 if (VariableOperand == -1) {
3723 AddrMode.BaseOffs += ConstantOffset;
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00003724 if (ConstantOffset == 0 ||
Mehdi Amini0cdec1e2015-07-09 02:09:40 +00003725 TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) {
Chandler Carruthc8925912013-01-05 02:09:22 +00003726 // Check to see if we can fold the base pointer in too.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003727 if (matchAddr(AddrInst->getOperand(0), Depth+1))
Chandler Carruthc8925912013-01-05 02:09:22 +00003728 return true;
3729 }
3730 AddrMode.BaseOffs -= ConstantOffset;
3731 return false;
3732 }
3733
3734 // Save the valid addressing mode in case we can't match.
3735 ExtAddrMode BackupAddrMode = AddrMode;
3736 unsigned OldSize = AddrModeInsts.size();
3737
3738 // See if the scale and offset amount is valid for this target.
3739 AddrMode.BaseOffs += ConstantOffset;
3740
3741 // Match the base operand of the GEP.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003742 if (!matchAddr(AddrInst->getOperand(0), Depth+1)) {
Chandler Carruthc8925912013-01-05 02:09:22 +00003743 // If it couldn't be matched, just stuff the value in a register.
3744 if (AddrMode.HasBaseReg) {
3745 AddrMode = BackupAddrMode;
3746 AddrModeInsts.resize(OldSize);
3747 return false;
3748 }
3749 AddrMode.HasBaseReg = true;
3750 AddrMode.BaseReg = AddrInst->getOperand(0);
3751 }
3752
3753 // Match the remaining variable portion of the GEP.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003754 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale,
Chandler Carruthc8925912013-01-05 02:09:22 +00003755 Depth)) {
3756 // If it couldn't be matched, try stuffing the base into a register
3757 // instead of matching it, and retrying the match of the scale.
3758 AddrMode = BackupAddrMode;
3759 AddrModeInsts.resize(OldSize);
3760 if (AddrMode.HasBaseReg)
3761 return false;
3762 AddrMode.HasBaseReg = true;
3763 AddrMode.BaseReg = AddrInst->getOperand(0);
3764 AddrMode.BaseOffs += ConstantOffset;
Sanjay Patelfc580a62015-09-21 23:03:16 +00003765 if (!matchScaledValue(AddrInst->getOperand(VariableOperand),
Chandler Carruthc8925912013-01-05 02:09:22 +00003766 VariableScale, Depth)) {
3767 // If even that didn't work, bail.
3768 AddrMode = BackupAddrMode;
3769 AddrModeInsts.resize(OldSize);
3770 return false;
3771 }
3772 }
3773
3774 return true;
3775 }
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003776 case Instruction::SExt:
3777 case Instruction::ZExt: {
3778 Instruction *Ext = dyn_cast<Instruction>(AddrInst);
3779 if (!Ext)
Sanjay Pateld3bbfa12014-07-16 22:40:28 +00003780 return false;
Sanjay Patelab60d042014-07-16 21:08:10 +00003781
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003782 // Try to move this ext out of the way of the addressing mode.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003783 // Ask for a method for doing so.
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003784 TypePromotionHelper::Action TPH =
Ahmed Bougachaf3299142015-06-17 20:44:32 +00003785 TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003786 if (!TPH)
3787 return false;
3788
3789 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3790 TPT.getRestorationPoint();
Quentin Colombet1b274f92015-03-10 21:48:15 +00003791 unsigned CreatedInstsCost = 0;
3792 unsigned ExtCost = !TLI.isExtFree(Ext);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00003793 Value *PromotedOperand =
Quentin Colombet1b274f92015-03-10 21:48:15 +00003794 TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003795 // SExt has been moved away.
3796 // Thus either it will be rematched later in the recursive calls or it is
3797 // gone. Anyway, we must not fold it into the addressing mode at this point.
3798 // E.g.,
3799 // op = add opnd, 1
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003800 // idx = ext op
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003801 // addr = gep base, idx
3802 // is now:
Quentin Colombetf5485bb2014-11-13 01:44:51 +00003803 // promotedOpnd = ext opnd <- no match here
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003804 // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls)
3805 // addr = gep base, op <- match
3806 if (MovedAway)
3807 *MovedAway = true;
3808
3809 assert(PromotedOperand &&
3810 "TypePromotionHelper should have filtered out those cases");
3811
3812 ExtAddrMode BackupAddrMode = AddrMode;
3813 unsigned OldSize = AddrModeInsts.size();
3814
Sanjay Patelfc580a62015-09-21 23:03:16 +00003815 if (!matchAddr(PromotedOperand, Depth) ||
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00003816 // The total of the new cost is equal to the cost of the created
Quentin Colombet1b274f92015-03-10 21:48:15 +00003817 // instructions.
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00003818 // The total of the old cost is equal to the cost of the extension plus
Quentin Colombet1b274f92015-03-10 21:48:15 +00003819 // what we have saved in the addressing mode.
Sanjay Patelfc580a62015-09-21 23:03:16 +00003820 !isPromotionProfitable(CreatedInstsCost,
Quentin Colombet1b274f92015-03-10 21:48:15 +00003821 ExtCost + (AddrModeInsts.size() - OldSize),
Quentin Colombet867c5502014-02-14 22:23:22 +00003822 PromotedOperand)) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003823 AddrMode = BackupAddrMode;
3824 AddrModeInsts.resize(OldSize);
3825 DEBUG(dbgs() << "Sign extension does not pay off: rollback\n");
3826 TPT.rollback(LastKnownGood);
3827 return false;
3828 }
3829 return true;
3830 }
Chandler Carruthc8925912013-01-05 02:09:22 +00003831 }
3832 return false;
3833}
3834
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003835/// If we can, try to add the value of 'Addr' into the current addressing mode.
3836/// If Addr can't be added to AddrMode this returns false and leaves AddrMode
3837/// unmodified. This assumes that Addr is either a pointer type or intptr_t
3838/// for the target.
Chandler Carruthc8925912013-01-05 02:09:22 +00003839///
Sanjay Patelfc580a62015-09-21 23:03:16 +00003840bool AddressingModeMatcher::matchAddr(Value *Addr, unsigned Depth) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003841 // Start a transaction at this point that we will rollback if the matching
3842 // fails.
3843 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3844 TPT.getRestorationPoint();
Chandler Carruthc8925912013-01-05 02:09:22 +00003845 if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {
3846 // Fold in immediates if legal for the target.
3847 AddrMode.BaseOffs += CI->getSExtValue();
Mehdi Amini0cdec1e2015-07-09 02:09:40 +00003848 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
Chandler Carruthc8925912013-01-05 02:09:22 +00003849 return true;
3850 AddrMode.BaseOffs -= CI->getSExtValue();
3851 } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {
3852 // If this is a global variable, try to fold it into the addressing mode.
Craig Topperc0196b12014-04-14 00:51:57 +00003853 if (!AddrMode.BaseGV) {
Chandler Carruthc8925912013-01-05 02:09:22 +00003854 AddrMode.BaseGV = GV;
Mehdi Amini0cdec1e2015-07-09 02:09:40 +00003855 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
Chandler Carruthc8925912013-01-05 02:09:22 +00003856 return true;
Craig Topperc0196b12014-04-14 00:51:57 +00003857 AddrMode.BaseGV = nullptr;
Chandler Carruthc8925912013-01-05 02:09:22 +00003858 }
3859 } else if (Instruction *I = dyn_cast<Instruction>(Addr)) {
3860 ExtAddrMode BackupAddrMode = AddrMode;
3861 unsigned OldSize = AddrModeInsts.size();
3862
3863 // Check to see if it is possible to fold this operation.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003864 bool MovedAway = false;
Sanjay Patelfc580a62015-09-21 23:03:16 +00003865 if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) {
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00003866 // This instruction may have been moved away. If so, there is nothing
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003867 // to check here.
3868 if (MovedAway)
3869 return true;
Chandler Carruthc8925912013-01-05 02:09:22 +00003870 // Okay, it's possible to fold this. Check to see if it is actually
3871 // *profitable* to do so. We use a simple cost model to avoid increasing
3872 // register pressure too much.
3873 if (I->hasOneUse() ||
Sanjay Patelfc580a62015-09-21 23:03:16 +00003874 isProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) {
Chandler Carruthc8925912013-01-05 02:09:22 +00003875 AddrModeInsts.push_back(I);
3876 return true;
3877 }
Stephen Lin837bba12013-07-15 17:55:02 +00003878
Chandler Carruthc8925912013-01-05 02:09:22 +00003879 // It isn't profitable to do this, roll back.
3880 //cerr << "NOT FOLDING: " << *I;
3881 AddrMode = BackupAddrMode;
3882 AddrModeInsts.resize(OldSize);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003883 TPT.rollback(LastKnownGood);
Chandler Carruthc8925912013-01-05 02:09:22 +00003884 }
3885 } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) {
Sanjay Patelfc580a62015-09-21 23:03:16 +00003886 if (matchOperationAddr(CE, CE->getOpcode(), Depth))
Chandler Carruthc8925912013-01-05 02:09:22 +00003887 return true;
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003888 TPT.rollback(LastKnownGood);
Chandler Carruthc8925912013-01-05 02:09:22 +00003889 } else if (isa<ConstantPointerNull>(Addr)) {
3890 // Null pointer gets folded without affecting the addressing mode.
3891 return true;
3892 }
3893
3894 // Worse case, the target should support [reg] addressing modes. :)
3895 if (!AddrMode.HasBaseReg) {
3896 AddrMode.HasBaseReg = true;
3897 AddrMode.BaseReg = Addr;
3898 // Still check for legality in case the target supports [imm] but not [i+r].
Mehdi Amini0cdec1e2015-07-09 02:09:40 +00003899 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
Chandler Carruthc8925912013-01-05 02:09:22 +00003900 return true;
3901 AddrMode.HasBaseReg = false;
Craig Topperc0196b12014-04-14 00:51:57 +00003902 AddrMode.BaseReg = nullptr;
Chandler Carruthc8925912013-01-05 02:09:22 +00003903 }
3904
3905 // If the base register is already taken, see if we can do [r+r].
3906 if (AddrMode.Scale == 0) {
3907 AddrMode.Scale = 1;
3908 AddrMode.ScaledReg = Addr;
Mehdi Amini0cdec1e2015-07-09 02:09:40 +00003909 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))
Chandler Carruthc8925912013-01-05 02:09:22 +00003910 return true;
3911 AddrMode.Scale = 0;
Craig Topperc0196b12014-04-14 00:51:57 +00003912 AddrMode.ScaledReg = nullptr;
Chandler Carruthc8925912013-01-05 02:09:22 +00003913 }
3914 // Couldn't match.
Quentin Colombet3a4bf042014-02-06 21:44:56 +00003915 TPT.rollback(LastKnownGood);
Chandler Carruthc8925912013-01-05 02:09:22 +00003916 return false;
3917}
3918
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003919/// Check to see if all uses of OpVal by the specified inline asm call are due
3920/// to memory operands. If so, return true, otherwise return false.
Chandler Carruthc8925912013-01-05 02:09:22 +00003921static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal,
Igor Laevsky3be81ba2017-02-07 13:27:20 +00003922 const TargetLowering &TLI,
3923 const TargetRegisterInfo &TRI) {
Sanjay Patel4137d512017-06-07 14:29:52 +00003924 const Function *F = CI->getFunction();
Eric Christopherd75c00c2015-02-26 22:38:34 +00003925 TargetLowering::AsmOperandInfoVector TargetConstraints =
Igor Laevsky3be81ba2017-02-07 13:27:20 +00003926 TLI.ParseConstraints(F->getParent()->getDataLayout(), &TRI,
Mehdi Amini8ac7a9d2015-07-07 19:07:19 +00003927 ImmutableCallSite(CI));
Igor Laevsky3be81ba2017-02-07 13:27:20 +00003928
Chandler Carruthc8925912013-01-05 02:09:22 +00003929 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
3930 TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
Stephen Lin837bba12013-07-15 17:55:02 +00003931
Chandler Carruthc8925912013-01-05 02:09:22 +00003932 // Compute the constraint code and ConstraintType to use.
Igor Laevsky3be81ba2017-02-07 13:27:20 +00003933 TLI.ComputeConstraintToUse(OpInfo, SDValue());
Chandler Carruthc8925912013-01-05 02:09:22 +00003934
3935 // If this asm operand is our Value*, and if it isn't an indirect memory
3936 // operand, we can't fold it!
3937 if (OpInfo.CallOperandVal == OpVal &&
3938 (OpInfo.ConstraintType != TargetLowering::C_Memory ||
3939 !OpInfo.isIndirect))
3940 return false;
3941 }
3942
3943 return true;
3944}
3945
Benjamin Kramerfc638c12017-07-24 16:18:09 +00003946// Max number of memory uses to look at before aborting the search to conserve
3947// compile time.
3948static constexpr int MaxMemoryUsesToScan = 20;
3949
Sanjay Patel4ac6b112015-09-21 22:47:23 +00003950/// Recursively walk all the uses of I until we find a memory use.
3951/// If we find an obviously non-foldable instruction, return true.
Chandler Carruthc8925912013-01-05 02:09:22 +00003952/// Add the ultimately found memory instructions to MemoryUses.
Eric Christopher11e4df72015-02-26 22:38:43 +00003953static bool FindAllMemoryUses(
3954 Instruction *I,
3955 SmallVectorImpl<std::pair<Instruction *, unsigned>> &MemoryUses,
Benjamin Kramerfc638c12017-07-24 16:18:09 +00003956 SmallPtrSetImpl<Instruction *> &ConsideredInsts, const TargetLowering &TLI,
3957 const TargetRegisterInfo &TRI, int SeenInsts = 0) {
Chandler Carruthc8925912013-01-05 02:09:22 +00003958 // If we already considered this instruction, we're done.
David Blaikie70573dc2014-11-19 07:49:26 +00003959 if (!ConsideredInsts.insert(I).second)
Chandler Carruthc8925912013-01-05 02:09:22 +00003960 return false;
Stephen Lin837bba12013-07-15 17:55:02 +00003961
Chandler Carruthc8925912013-01-05 02:09:22 +00003962 // If this is an obviously unfoldable instruction, bail out.
3963 if (!MightBeFoldableInst(I))
3964 return true;
3965
Philip Reamesac115ed2016-03-09 23:13:12 +00003966 const bool OptSize = I->getFunction()->optForSize();
3967
Chandler Carruthc8925912013-01-05 02:09:22 +00003968 // Loop over all the uses, recursively processing them.
Chandler Carruthcdf47882014-03-09 03:16:01 +00003969 for (Use &U : I->uses()) {
Benjamin Kramerfc638c12017-07-24 16:18:09 +00003970 // Conservatively return true if we're seeing a large number or a deep chain
3971 // of users. This avoids excessive compilation times in pathological cases.
3972 if (SeenInsts++ >= MaxMemoryUsesToScan)
3973 return true;
Chandler Carruthc8925912013-01-05 02:09:22 +00003974
Benjamin Kramerfc638c12017-07-24 16:18:09 +00003975 Instruction *UserI = cast<Instruction>(U.getUser());
Chandler Carruthcdf47882014-03-09 03:16:01 +00003976 if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) {
3977 MemoryUses.push_back(std::make_pair(LI, U.getOperandNo()));
Chandler Carruthc8925912013-01-05 02:09:22 +00003978 continue;
3979 }
Stephen Lin837bba12013-07-15 17:55:02 +00003980
Chandler Carruthcdf47882014-03-09 03:16:01 +00003981 if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) {
3982 unsigned opNo = U.getOperandNo();
Matt Arsenault02d915b2017-03-15 22:35:20 +00003983 if (opNo != StoreInst::getPointerOperandIndex())
3984 return true; // Storing addr, not into addr.
Chandler Carruthc8925912013-01-05 02:09:22 +00003985 MemoryUses.push_back(std::make_pair(SI, opNo));
3986 continue;
3987 }
Stephen Lin837bba12013-07-15 17:55:02 +00003988
Matt Arsenault02d915b2017-03-15 22:35:20 +00003989 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(UserI)) {
3990 unsigned opNo = U.getOperandNo();
3991 if (opNo != AtomicRMWInst::getPointerOperandIndex())
3992 return true; // Storing addr, not into addr.
3993 MemoryUses.push_back(std::make_pair(RMW, opNo));
3994 continue;
3995 }
3996
3997 if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(UserI)) {
3998 unsigned opNo = U.getOperandNo();
3999 if (opNo != AtomicCmpXchgInst::getPointerOperandIndex())
4000 return true; // Storing addr, not into addr.
4001 MemoryUses.push_back(std::make_pair(CmpX, opNo));
4002 continue;
4003 }
4004
Chandler Carruthcdf47882014-03-09 03:16:01 +00004005 if (CallInst *CI = dyn_cast<CallInst>(UserI)) {
Philip Reamesac115ed2016-03-09 23:13:12 +00004006 // If this is a cold call, we can sink the addressing calculation into
4007 // the cold path. See optimizeCallInst
4008 if (!OptSize && CI->hasFnAttr(Attribute::Cold))
4009 continue;
Junmo Park6098cbb2016-03-11 07:05:32 +00004010
Chandler Carruthc8925912013-01-05 02:09:22 +00004011 InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue());
4012 if (!IA) return true;
Stephen Lin837bba12013-07-15 17:55:02 +00004013
Chandler Carruthc8925912013-01-05 02:09:22 +00004014 // If this is a memory operand, we're cool, otherwise bail out.
Igor Laevsky3be81ba2017-02-07 13:27:20 +00004015 if (!IsOperandAMemoryOperand(CI, IA, I, TLI, TRI))
Chandler Carruthc8925912013-01-05 02:09:22 +00004016 return true;
4017 continue;
4018 }
Stephen Lin837bba12013-07-15 17:55:02 +00004019
Benjamin Kramerfc638c12017-07-24 16:18:09 +00004020 if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TLI, TRI,
4021 SeenInsts))
Chandler Carruthc8925912013-01-05 02:09:22 +00004022 return true;
4023 }
4024
4025 return false;
4026}
4027
Sanjay Patel9fbe22b2015-10-09 18:01:03 +00004028/// Return true if Val is already known to be live at the use site that we're
4029/// folding it into. If so, there is no cost to include it in the addressing
4030/// mode. KnownLive1 and KnownLive2 are two values that we know are live at the
4031/// instruction already.
Sanjay Patelfc580a62015-09-21 23:03:16 +00004032bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value *Val,Value *KnownLive1,
Chandler Carruthc8925912013-01-05 02:09:22 +00004033 Value *KnownLive2) {
4034 // If Val is either of the known-live values, we know it is live!
Craig Topperc0196b12014-04-14 00:51:57 +00004035 if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2)
Chandler Carruthc8925912013-01-05 02:09:22 +00004036 return true;
Stephen Lin837bba12013-07-15 17:55:02 +00004037
Chandler Carruthc8925912013-01-05 02:09:22 +00004038 // All values other than instructions and arguments (e.g. constants) are live.
4039 if (!isa<Instruction>(Val) && !isa<Argument>(Val)) return true;
Stephen Lin837bba12013-07-15 17:55:02 +00004040
Chandler Carruthc8925912013-01-05 02:09:22 +00004041 // If Val is a constant sized alloca in the entry block, it is live, this is
4042 // true because it is just a reference to the stack/frame pointer, which is
4043 // live for the whole function.
4044 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val))
4045 if (AI->isStaticAlloca())
4046 return true;
Stephen Lin837bba12013-07-15 17:55:02 +00004047
Chandler Carruthc8925912013-01-05 02:09:22 +00004048 // Check to see if this value is already used in the memory instruction's
4049 // block. If so, it's already live into the block at the very least, so we
4050 // can reasonably fold it.
4051 return Val->isUsedInBasicBlock(MemoryInst->getParent());
4052}
4053
Sanjay Patel4ac6b112015-09-21 22:47:23 +00004054/// It is possible for the addressing mode of the machine to fold the specified
4055/// instruction into a load or store that ultimately uses it.
4056/// However, the specified instruction has multiple uses.
4057/// Given this, it may actually increase register pressure to fold it
4058/// into the load. For example, consider this code:
Chandler Carruthc8925912013-01-05 02:09:22 +00004059///
4060/// X = ...
4061/// Y = X+1
4062/// use(Y) -> nonload/store
4063/// Z = Y+1
4064/// load Z
4065///
4066/// In this case, Y has multiple uses, and can be folded into the load of Z
4067/// (yielding load [X+2]). However, doing this will cause both "X" and "X+1" to
4068/// be live at the use(Y) line. If we don't fold Y into load Z, we use one
4069/// fewer register. Since Y can't be folded into "use(Y)" we don't increase the
4070/// number of computations either.
4071///
4072/// Note that this (like most of CodeGenPrepare) is just a rough heuristic. If
4073/// X was live across 'load Z' for other reasons, we actually *would* want to
4074/// fold the addressing mode in the Z case. This would make Y die earlier.
4075bool AddressingModeMatcher::
Sanjay Patelfc580a62015-09-21 23:03:16 +00004076isProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore,
Chandler Carruthc8925912013-01-05 02:09:22 +00004077 ExtAddrMode &AMAfter) {
4078 if (IgnoreProfitability) return true;
Stephen Lin837bba12013-07-15 17:55:02 +00004079
Chandler Carruthc8925912013-01-05 02:09:22 +00004080 // AMBefore is the addressing mode before this instruction was folded into it,
4081 // and AMAfter is the addressing mode after the instruction was folded. Get
4082 // the set of registers referenced by AMAfter and subtract out those
4083 // referenced by AMBefore: this is the set of values which folding in this
4084 // address extends the lifetime of.
4085 //
4086 // Note that there are only two potential values being referenced here,
4087 // BaseReg and ScaleReg (global addresses are always available, as are any
4088 // folded immediates).
4089 Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg;
Stephen Lin837bba12013-07-15 17:55:02 +00004090
Chandler Carruthc8925912013-01-05 02:09:22 +00004091 // If the BaseReg or ScaledReg was referenced by the previous addrmode, their
4092 // lifetime wasn't extended by adding this instruction.
Sanjay Patelfc580a62015-09-21 23:03:16 +00004093 if (valueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg))
Craig Topperc0196b12014-04-14 00:51:57 +00004094 BaseReg = nullptr;
Sanjay Patelfc580a62015-09-21 23:03:16 +00004095 if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg))
Craig Topperc0196b12014-04-14 00:51:57 +00004096 ScaledReg = nullptr;
Chandler Carruthc8925912013-01-05 02:09:22 +00004097
4098 // If folding this instruction (and it's subexprs) didn't extend any live
4099 // ranges, we're ok with it.
Craig Topperc0196b12014-04-14 00:51:57 +00004100 if (!BaseReg && !ScaledReg)
Chandler Carruthc8925912013-01-05 02:09:22 +00004101 return true;
4102
Philip Reamesac115ed2016-03-09 23:13:12 +00004103 // If all uses of this instruction can have the address mode sunk into them,
4104 // we can remove the addressing mode and effectively trade one live register
4105 // for another (at worst.) In this context, folding an addressing mode into
Junmo Park6098cbb2016-03-11 07:05:32 +00004106 // the use is just a particularly nice way of sinking it.
Chandler Carruthc8925912013-01-05 02:09:22 +00004107 SmallVector<std::pair<Instruction*,unsigned>, 16> MemoryUses;
4108 SmallPtrSet<Instruction*, 16> ConsideredInsts;
Igor Laevsky3be81ba2017-02-07 13:27:20 +00004109 if (FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TLI, TRI))
Chandler Carruthc8925912013-01-05 02:09:22 +00004110 return false; // Has a non-memory, non-foldable use!
Stephen Lin837bba12013-07-15 17:55:02 +00004111
Chandler Carruthc8925912013-01-05 02:09:22 +00004112 // Now that we know that all uses of this instruction are part of a chain of
4113 // computation involving only operations that could theoretically be folded
Philip Reamesac115ed2016-03-09 23:13:12 +00004114 // into a memory use, loop over each of these memory operation uses and see
4115 // if they could *actually* fold the instruction. The assumption is that
4116 // addressing modes are cheap and that duplicating the computation involved
4117 // many times is worthwhile, even on a fastpath. For sinking candidates
4118 // (i.e. cold call sites), this serves as a way to prevent excessive code
4119 // growth since most architectures have some reasonable small and fast way to
4120 // compute an effective address. (i.e LEA on x86)
Chandler Carruthc8925912013-01-05 02:09:22 +00004121 SmallVector<Instruction*, 32> MatchedAddrModeInsts;
4122 for (unsigned i = 0, e = MemoryUses.size(); i != e; ++i) {
4123 Instruction *User = MemoryUses[i].first;
4124 unsigned OpNo = MemoryUses[i].second;
Stephen Lin837bba12013-07-15 17:55:02 +00004125
Chandler Carruthc8925912013-01-05 02:09:22 +00004126 // Get the access type of this use. If the use isn't a pointer, we don't
4127 // know what it accesses.
4128 Value *Address = User->getOperand(OpNo);
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00004129 PointerType *AddrTy = dyn_cast<PointerType>(Address->getType());
4130 if (!AddrTy)
Chandler Carruthc8925912013-01-05 02:09:22 +00004131 return false;
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00004132 Type *AddressAccessTy = AddrTy->getElementType();
4133 unsigned AS = AddrTy->getAddressSpace();
Stephen Lin837bba12013-07-15 17:55:02 +00004134
Chandler Carruthc8925912013-01-05 02:09:22 +00004135 // Do a match against the root of this address, ignoring profitability. This
4136 // will tell us if the addressing mode for the memory operation will
4137 // *actually* cover the shared instruction.
4138 ExtAddrMode Result;
Quentin Colombet5a69dda2014-02-11 01:59:02 +00004139 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4140 TPT.getRestorationPoint();
Igor Laevsky3be81ba2017-02-07 13:27:20 +00004141 AddressingModeMatcher Matcher(MatchedAddrModeInsts, TLI, TRI,
4142 AddressAccessTy, AS,
Ahmed Bougachaf3299142015-06-17 20:44:32 +00004143 MemoryInst, Result, InsertedInsts,
Quentin Colombet3a4bf042014-02-06 21:44:56 +00004144 PromotedInsts, TPT);
Chandler Carruthc8925912013-01-05 02:09:22 +00004145 Matcher.IgnoreProfitability = true;
Sanjay Patelfc580a62015-09-21 23:03:16 +00004146 bool Success = Matcher.matchAddr(Address, 0);
Chandler Carruthc8925912013-01-05 02:09:22 +00004147 (void)Success; assert(Success && "Couldn't select *anything*?");
4148
Quentin Colombet5a69dda2014-02-11 01:59:02 +00004149 // The match was to check the profitability, the changes made are not
4150 // part of the original matcher. Therefore, they should be dropped
4151 // otherwise the original matcher will not present the right state.
4152 TPT.rollback(LastKnownGood);
4153
Chandler Carruthc8925912013-01-05 02:09:22 +00004154 // If the match didn't cover I, then it won't be shared by it.
David Majnemer0d955d02016-08-11 22:21:41 +00004155 if (!is_contained(MatchedAddrModeInsts, I))
Chandler Carruthc8925912013-01-05 02:09:22 +00004156 return false;
Stephen Lin837bba12013-07-15 17:55:02 +00004157
Chandler Carruthc8925912013-01-05 02:09:22 +00004158 MatchedAddrModeInsts.clear();
4159 }
Stephen Lin837bba12013-07-15 17:55:02 +00004160
Chandler Carruthc8925912013-01-05 02:09:22 +00004161 return true;
4162}
4163
Sanjay Patel4ac6b112015-09-21 22:47:23 +00004164/// Return true if the specified values are defined in a
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004165/// different basic block than BB.
4166static bool IsNonLocalValue(Value *V, BasicBlock *BB) {
4167 if (Instruction *I = dyn_cast<Instruction>(V))
4168 return I->getParent() != BB;
4169 return false;
4170}
4171
Philip Reamesac115ed2016-03-09 23:13:12 +00004172/// Sink addressing mode computation immediate before MemoryInst if doing so
4173/// can be done without increasing register pressure. The need for the
4174/// register pressure constraint means this can end up being an all or nothing
4175/// decision for all uses of the same addressing computation.
4176///
Sanjay Patel4ac6b112015-09-21 22:47:23 +00004177/// Load and Store Instructions often have addressing modes that can do
4178/// significant amounts of computation. As such, instruction selection will try
4179/// to get the load or store to do as much computation as possible for the
4180/// program. The problem is that isel can only see within a single block. As
4181/// such, we sink as much legal addressing mode work into the block as possible.
Chris Lattner728f9022008-11-25 07:09:13 +00004182///
4183/// This method is used to optimize both load/store and inline asms with memory
Philip Reamesac115ed2016-03-09 23:13:12 +00004184/// operands. It's also used to sink addressing computations feeding into cold
4185/// call sites into their (cold) basic block.
4186///
4187/// The motivation for handling sinking into cold blocks is that doing so can
4188/// both enable other address mode sinking (by satisfying the register pressure
4189/// constraint above), and reduce register pressure globally (by removing the
4190/// addressing mode computation from the fast path entirely.).
Sanjay Patelfc580a62015-09-21 23:03:16 +00004191bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00004192 Type *AccessTy, unsigned AddrSpace) {
Owen Anderson8ba5f392010-11-27 08:15:55 +00004193 Value *Repl = Addr;
Nadav Rotem465834c2012-07-24 10:51:42 +00004194
4195 // Try to collapse single-value PHI nodes. This is necessary to undo
Owen Andersondfb8c3b2010-11-19 22:15:03 +00004196 // unprofitable PRE transformations.
Cameron Zwarich43cecb12011-01-03 06:33:01 +00004197 SmallVector<Value*, 8> worklist;
4198 SmallPtrSet<Value*, 16> Visited;
Owen Anderson8ba5f392010-11-27 08:15:55 +00004199 worklist.push_back(Addr);
Nadav Rotem465834c2012-07-24 10:51:42 +00004200
John Brawneb83c752017-10-03 13:04:15 +00004201 // Use a worklist to iteratively look through PHI and select nodes, and
4202 // ensure that the addressing mode obtained from the non-PHI/select roots of
John Brawn736bf002017-10-03 13:08:22 +00004203 // the graph are compatible.
John Brawneb83c752017-10-03 13:04:15 +00004204 bool PhiOrSelectSeen = false;
Owen Anderson8ba5f392010-11-27 08:15:55 +00004205 SmallVector<Instruction*, 16> AddrModeInsts;
Serguei Katkovaee63752017-11-05 07:59:02 +00004206 const SimplifyQuery SQ(*DL, TLInfo);
4207 AddressingModeCombiner AddrModes(SQ, { Addr, MemoryInst->getParent() });
Jun Bum Limdee55652017-04-03 19:20:07 +00004208 TypePromotionTransaction TPT(RemovedInsts);
Quentin Colombet3a4bf042014-02-06 21:44:56 +00004209 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4210 TPT.getRestorationPoint();
Owen Anderson8ba5f392010-11-27 08:15:55 +00004211 while (!worklist.empty()) {
4212 Value *V = worklist.back();
4213 worklist.pop_back();
Nadav Rotem465834c2012-07-24 10:51:42 +00004214
Serguei Katkov4ea855e2017-07-19 04:49:17 +00004215 // We allow traversing cyclic Phi nodes.
4216 // In case of success after this loop we ensure that traversing through
4217 // Phi nodes ends up with all cases to compute address of the form
4218 // BaseGV + Base + Scale * Index + Offset
4219 // where Scale and Offset are constans and BaseGV, Base and Index
4220 // are exactly the same Values in all cases.
4221 // It means that BaseGV, Scale and Offset dominate our memory instruction
4222 // and have the same value as they had in address computation represented
4223 // as Phi. So we can safely sink address computation to memory instruction.
4224 if (!Visited.insert(V).second)
4225 continue;
Nadav Rotem465834c2012-07-24 10:51:42 +00004226
Owen Anderson8ba5f392010-11-27 08:15:55 +00004227 // For a PHI node, push all of its incoming values.
4228 if (PHINode *P = dyn_cast<PHINode>(V)) {
Pete Cooper833f34d2015-05-12 20:05:31 +00004229 for (Value *IncValue : P->incoming_values())
4230 worklist.push_back(IncValue);
John Brawneb83c752017-10-03 13:04:15 +00004231 PhiOrSelectSeen = true;
4232 continue;
4233 }
4234 // Similar for select.
4235 if (SelectInst *SI = dyn_cast<SelectInst>(V)) {
4236 worklist.push_back(SI->getFalseValue());
4237 worklist.push_back(SI->getTrueValue());
4238 PhiOrSelectSeen = true;
Owen Anderson8ba5f392010-11-27 08:15:55 +00004239 continue;
4240 }
Nadav Rotem465834c2012-07-24 10:51:42 +00004241
Philip Reamesac115ed2016-03-09 23:13:12 +00004242 // For non-PHIs, determine the addressing mode being computed. Note that
4243 // the result may differ depending on what other uses our candidate
4244 // addressing instructions might have.
Serguei Katkova6fba3d2017-07-18 05:16:38 +00004245 AddrModeInsts.clear();
Quentin Colombet3a4bf042014-02-06 21:44:56 +00004246 ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
Serguei Katkova6fba3d2017-07-18 05:16:38 +00004247 V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *TLI, *TRI,
4248 InsertedInsts, PromotedInsts, TPT);
John Brawn736bf002017-10-03 13:08:22 +00004249 NewAddrMode.OriginalValue = V;
Cameron Zwarich13c885d2011-03-05 08:12:26 +00004250
John Brawn736bf002017-10-03 13:08:22 +00004251 if (!AddrModes.addNewAddrMode(NewAddrMode))
4252 break;
Owen Andersondfb8c3b2010-11-19 22:15:03 +00004253 }
Nadav Rotem465834c2012-07-24 10:51:42 +00004254
John Brawn736bf002017-10-03 13:08:22 +00004255 // Try to combine the AddrModes we've collected. If we couldn't collect any,
4256 // or we have multiple but either couldn't combine them or combining them
4257 // wouldn't do anything useful, bail out now.
4258 if (!AddrModes.combineAddrModes()) {
Quentin Colombet3a4bf042014-02-06 21:44:56 +00004259 TPT.rollback(LastKnownGood);
4260 return false;
4261 }
4262 TPT.commit();
Nadav Rotem465834c2012-07-24 10:51:42 +00004263
John Brawn736bf002017-10-03 13:08:22 +00004264 // Get the combined AddrMode (or the only AddrMode, if we only had one).
4265 ExtAddrMode AddrMode = AddrModes.getAddrMode();
4266
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004267 // If all the instructions matched are already in this BB, don't do anything.
John Brawneb83c752017-10-03 13:04:15 +00004268 // If we saw a Phi node then it is not local definitely, and if we saw a select
4269 // then we want to push the address calculation past it even if it's already
4270 // in this BB.
4271 if (!PhiOrSelectSeen && none_of(AddrModeInsts, [&](Value *V) {
Justin Lebar838c7f52016-11-21 22:49:11 +00004272 return IsNonLocalValue(V, MemoryInst->getParent());
Serguei Katkov0b7b59a2017-07-11 06:24:44 +00004273 })) {
David Greene74e2d492010-01-05 01:27:11 +00004274 DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode << "\n");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004275 return false;
4276 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00004277
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004278 // Insert this computation right after this user. Since our caller is
4279 // scanning from the top of the BB to the bottom, reuse of the expr are
4280 // guaranteed to happen later.
Devang Patelc10e52a2011-09-06 18:49:53 +00004281 IRBuilder<> Builder(MemoryInst);
Eric Christopherc1ea1492008-09-24 05:32:41 +00004282
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004283 // Now that we determined the addressing expression we want to use and know
4284 // that we have to sink it into this block. Check to see if we have already
Simon Dardis230f4532017-11-24 16:45:28 +00004285 // done this for some other load/store instr in this block. If so, reuse
4286 // the computation. Before attempting reuse, check if the address is valid
4287 // as it may have been erased.
4288
4289 WeakTrackingVH SunkAddrVH = SunkAddrs[Addr];
4290
4291 Value * SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004292 if (SunkAddr) {
David Greene74e2d492010-01-05 01:27:11 +00004293 DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode << " for "
Louis Gerbarg1b91aa22014-05-13 21:54:22 +00004294 << *MemoryInst << "\n");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004295 if (SunkAddr->getType() != Addr->getType())
Eli Friedmanc12a5a72017-02-24 20:51:36 +00004296 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType());
Eric Christopherfccff372015-01-27 01:01:38 +00004297 } else if (AddrSinkUsingGEPs ||
4298 (!AddrSinkUsingGEPs.getNumOccurrences() && TM &&
Igor Laevsky3be81ba2017-02-07 13:27:20 +00004299 SubtargetInfo->useAA())) {
Hal Finkelc3998302014-04-12 00:59:48 +00004300 // By default, we use the GEP-based method when AA is used later. This
4301 // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities.
4302 DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
Louis Gerbarg1b91aa22014-05-13 21:54:22 +00004303 << *MemoryInst << "\n");
Mehdi Amini4fe37982015-07-07 18:45:17 +00004304 Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
Craig Topperc0196b12014-04-14 00:51:57 +00004305 Value *ResultPtr = nullptr, *ResultIndex = nullptr;
Hal Finkelc3998302014-04-12 00:59:48 +00004306
4307 // First, find the pointer.
4308 if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) {
4309 ResultPtr = AddrMode.BaseReg;
Craig Topperc0196b12014-04-14 00:51:57 +00004310 AddrMode.BaseReg = nullptr;
Hal Finkelc3998302014-04-12 00:59:48 +00004311 }
4312
4313 if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) {
4314 // We can't add more than one pointer together, nor can we scale a
4315 // pointer (both of which seem meaningless).
4316 if (ResultPtr || AddrMode.Scale != 1)
4317 return false;
4318
4319 ResultPtr = AddrMode.ScaledReg;
4320 AddrMode.Scale = 0;
4321 }
4322
Eli Friedman6f7c9ad2017-07-12 23:30:02 +00004323 // It is only safe to sign extend the BaseReg if we know that the math
4324 // required to create it did not overflow before we extend it. Since
4325 // the original IR value was tossed in favor of a constant back when
4326 // the AddrMode was created we need to bail out gracefully if widths
4327 // do not match instead of extending it.
4328 //
4329 // (See below for code to add the scale.)
4330 if (AddrMode.Scale) {
4331 Type *ScaledRegTy = AddrMode.ScaledReg->getType();
4332 if (cast<IntegerType>(IntPtrTy)->getBitWidth() >
4333 cast<IntegerType>(ScaledRegTy)->getBitWidth())
4334 return false;
4335 }
4336
Hal Finkelc3998302014-04-12 00:59:48 +00004337 if (AddrMode.BaseGV) {
4338 if (ResultPtr)
4339 return false;
4340
4341 ResultPtr = AddrMode.BaseGV;
4342 }
4343
4344 // If the real base value actually came from an inttoptr, then the matcher
4345 // will look through it and provide only the integer value. In that case,
4346 // use it here.
Keno Fischer05e4ac22017-06-29 20:28:59 +00004347 if (!DL->isNonIntegralPointerType(Addr->getType())) {
4348 if (!ResultPtr && AddrMode.BaseReg) {
4349 ResultPtr = Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(),
4350 "sunkaddr");
4351 AddrMode.BaseReg = nullptr;
4352 } else if (!ResultPtr && AddrMode.Scale == 1) {
4353 ResultPtr = Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(),
4354 "sunkaddr");
4355 AddrMode.Scale = 0;
4356 }
Hal Finkelc3998302014-04-12 00:59:48 +00004357 }
4358
4359 if (!ResultPtr &&
4360 !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) {
4361 SunkAddr = Constant::getNullValue(Addr->getType());
4362 } else if (!ResultPtr) {
4363 return false;
4364 } else {
4365 Type *I8PtrTy =
David Blaikie3909da72015-03-30 20:42:56 +00004366 Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace());
4367 Type *I8Ty = Builder.getInt8Ty();
Hal Finkelc3998302014-04-12 00:59:48 +00004368
4369 // Start with the base register. Do this first so that subsequent address
4370 // matching finds it last, which will prevent it from trying to match it
4371 // as the scaled value in case it happens to be a mul. That would be
4372 // problematic if we've sunk a different mul for the scale, because then
4373 // we'd end up sinking both muls.
4374 if (AddrMode.BaseReg) {
4375 Value *V = AddrMode.BaseReg;
4376 if (V->getType() != IntPtrTy)
4377 V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");
4378
4379 ResultIndex = V;
4380 }
4381
4382 // Add the scale value.
4383 if (AddrMode.Scale) {
4384 Value *V = AddrMode.ScaledReg;
4385 if (V->getType() == IntPtrTy) {
4386 // done.
Hal Finkelc3998302014-04-12 00:59:48 +00004387 } else {
Eli Friedman6f7c9ad2017-07-12 23:30:02 +00004388 assert(cast<IntegerType>(IntPtrTy)->getBitWidth() <
4389 cast<IntegerType>(V->getType())->getBitWidth() &&
4390 "We can't transform if ScaledReg is too narrow");
4391 V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");
Hal Finkelc3998302014-04-12 00:59:48 +00004392 }
4393
4394 if (AddrMode.Scale != 1)
4395 V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
4396 "sunkaddr");
4397 if (ResultIndex)
4398 ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr");
4399 else
4400 ResultIndex = V;
4401 }
4402
4403 // Add in the Base Offset if present.
4404 if (AddrMode.BaseOffs) {
4405 Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs);
4406 if (ResultIndex) {
NAKAMURA Takumif51a34e2014-10-29 15:23:11 +00004407 // We need to add this separately from the scale above to help with
4408 // SDAG consecutive load/store merging.
Hal Finkelc3998302014-04-12 00:59:48 +00004409 if (ResultPtr->getType() != I8PtrTy)
Eli Friedmanc12a5a72017-02-24 20:51:36 +00004410 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);
David Blaikie3909da72015-03-30 20:42:56 +00004411 ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr");
Hal Finkelc3998302014-04-12 00:59:48 +00004412 }
4413
4414 ResultIndex = V;
4415 }
4416
4417 if (!ResultIndex) {
4418 SunkAddr = ResultPtr;
4419 } else {
4420 if (ResultPtr->getType() != I8PtrTy)
Eli Friedmanc12a5a72017-02-24 20:51:36 +00004421 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);
David Blaikie3909da72015-03-30 20:42:56 +00004422 SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr");
Hal Finkelc3998302014-04-12 00:59:48 +00004423 }
4424
4425 if (SunkAddr->getType() != Addr->getType())
Eli Friedmanc12a5a72017-02-24 20:51:36 +00004426 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType());
Hal Finkelc3998302014-04-12 00:59:48 +00004427 }
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004428 } else {
Keno Fischer05e4ac22017-06-29 20:28:59 +00004429 // We'd require a ptrtoint/inttoptr down the line, which we can't do for
4430 // non-integral pointers, so in that case bail out now.
4431 Type *BaseTy = AddrMode.BaseReg ? AddrMode.BaseReg->getType() : nullptr;
4432 Type *ScaleTy = AddrMode.Scale ? AddrMode.ScaledReg->getType() : nullptr;
4433 PointerType *BasePtrTy = dyn_cast_or_null<PointerType>(BaseTy);
4434 PointerType *ScalePtrTy = dyn_cast_or_null<PointerType>(ScaleTy);
4435 if (DL->isNonIntegralPointerType(Addr->getType()) ||
4436 (BasePtrTy && DL->isNonIntegralPointerType(BasePtrTy)) ||
4437 (ScalePtrTy && DL->isNonIntegralPointerType(ScalePtrTy)) ||
4438 (AddrMode.BaseGV &&
4439 DL->isNonIntegralPointerType(AddrMode.BaseGV->getType())))
4440 return false;
4441
David Greene74e2d492010-01-05 01:27:11 +00004442 DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
Louis Gerbarg1b91aa22014-05-13 21:54:22 +00004443 << *MemoryInst << "\n");
Mehdi Amini4fe37982015-07-07 18:45:17 +00004444 Type *IntPtrTy = DL->getIntPtrType(Addr->getType());
Craig Topperc0196b12014-04-14 00:51:57 +00004445 Value *Result = nullptr;
Dan Gohmanca194452010-01-19 22:45:06 +00004446
4447 // Start with the base register. Do this first so that subsequent address
4448 // matching finds it last, which will prevent it from trying to match it
4449 // as the scaled value in case it happens to be a mul. That would be
4450 // problematic if we've sunk a different mul for the scale, because then
4451 // we'd end up sinking both muls.
4452 if (AddrMode.BaseReg) {
4453 Value *V = AddrMode.BaseReg;
Duncan Sands19d0b472010-02-16 11:11:14 +00004454 if (V->getType()->isPointerTy())
Devang Patelc10e52a2011-09-06 18:49:53 +00004455 V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
Dan Gohmanca194452010-01-19 22:45:06 +00004456 if (V->getType() != IntPtrTy)
Devang Patelc10e52a2011-09-06 18:49:53 +00004457 V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");
Dan Gohmanca194452010-01-19 22:45:06 +00004458 Result = V;
4459 }
4460
4461 // Add the scale value.
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004462 if (AddrMode.Scale) {
4463 Value *V = AddrMode.ScaledReg;
4464 if (V->getType() == IntPtrTy) {
4465 // done.
Duncan Sands19d0b472010-02-16 11:11:14 +00004466 } else if (V->getType()->isPointerTy()) {
Devang Patelc10e52a2011-09-06 18:49:53 +00004467 V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004468 } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <
4469 cast<IntegerType>(V->getType())->getBitWidth()) {
Devang Patelc10e52a2011-09-06 18:49:53 +00004470 V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004471 } else {
Jim Grosbached2cd392014-03-26 17:27:01 +00004472 // It is only safe to sign extend the BaseReg if we know that the math
4473 // required to create it did not overflow before we extend it. Since
4474 // the original IR value was tossed in favor of a constant back when
4475 // the AddrMode was created we need to bail out gracefully if widths
4476 // do not match instead of extending it.
Joey Gouly12a8bf02014-05-13 15:42:45 +00004477 Instruction *I = dyn_cast_or_null<Instruction>(Result);
Jim Grosbach83b44e12014-04-10 00:27:45 +00004478 if (I && (Result != AddrMode.BaseReg))
4479 I->eraseFromParent();
Jim Grosbached2cd392014-03-26 17:27:01 +00004480 return false;
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004481 }
4482 if (AddrMode.Scale != 1)
Devang Patelc10e52a2011-09-06 18:49:53 +00004483 V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),
4484 "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004485 if (Result)
Devang Patelc10e52a2011-09-06 18:49:53 +00004486 Result = Builder.CreateAdd(Result, V, "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004487 else
4488 Result = V;
4489 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00004490
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004491 // Add in the BaseGV if present.
4492 if (AddrMode.BaseGV) {
Devang Patelc10e52a2011-09-06 18:49:53 +00004493 Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004494 if (Result)
Devang Patelc10e52a2011-09-06 18:49:53 +00004495 Result = Builder.CreateAdd(Result, V, "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004496 else
4497 Result = V;
4498 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00004499
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004500 // Add in the Base Offset if present.
4501 if (AddrMode.BaseOffs) {
Owen Andersonedb4a702009-07-24 23:12:02 +00004502 Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs);
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004503 if (Result)
Devang Patelc10e52a2011-09-06 18:49:53 +00004504 Result = Builder.CreateAdd(Result, V, "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004505 else
4506 Result = V;
4507 }
4508
Craig Topperc0196b12014-04-14 00:51:57 +00004509 if (!Result)
Owen Anderson5a1acd92009-07-31 20:28:14 +00004510 SunkAddr = Constant::getNullValue(Addr->getType());
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004511 else
Devang Patelc10e52a2011-09-06 18:49:53 +00004512 SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr");
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004513 }
Eric Christopherc1ea1492008-09-24 05:32:41 +00004514
Owen Andersondfb8c3b2010-11-19 22:15:03 +00004515 MemoryInst->replaceUsesOfWith(Repl, SunkAddr);
Simon Dardis230f4532017-11-24 16:45:28 +00004516 // Store the newly computed address into the cache. In the case we reused a
4517 // value, this should be idempotent.
4518 SunkAddrs[Addr] = WeakTrackingVH(SunkAddr);
Eric Christopherc1ea1492008-09-24 05:32:41 +00004519
Chris Lattneraf1bcce2011-04-09 07:05:44 +00004520 // If we have no uses, recursively delete the value and all dead instructions
4521 // using it.
Owen Andersondfb8c3b2010-11-19 22:15:03 +00004522 if (Repl->use_empty()) {
Chris Lattneraf1bcce2011-04-09 07:05:44 +00004523 // This can cause recursive deletion, which can invalidate our iterator.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00004524 // Use a WeakTrackingVH to hold onto it in case this happens.
Duncan P. N. Exon Smith7b269642016-02-21 19:37:45 +00004525 Value *CurValue = &*CurInstIterator;
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00004526 WeakTrackingVH IterHandle(CurValue);
Chris Lattneraf1bcce2011-04-09 07:05:44 +00004527 BasicBlock *BB = CurInstIterator->getParent();
Nadav Rotem465834c2012-07-24 10:51:42 +00004528
Benjamin Kramer8bcc9712012-08-29 15:32:21 +00004529 RecursivelyDeleteTriviallyDeadInstructions(Repl, TLInfo);
Chris Lattneraf1bcce2011-04-09 07:05:44 +00004530
Duncan P. N. Exon Smith7b269642016-02-21 19:37:45 +00004531 if (IterHandle != CurValue) {
Chris Lattneraf1bcce2011-04-09 07:05:44 +00004532 // If the iterator instruction was recursively deleted, start over at the
4533 // start of the block.
4534 CurInstIterator = BB->begin();
4535 SunkAddrs.clear();
Nadav Rotem465834c2012-07-24 10:51:42 +00004536 }
Dale Johannesenb67a6e662010-03-31 20:37:15 +00004537 }
Cameron Zwarichced753f2011-01-05 17:27:27 +00004538 ++NumMemoryInsts;
Chris Lattnerfeee64e2007-04-13 20:30:56 +00004539 return true;
4540}
4541
Sanjay Patel4ac6b112015-09-21 22:47:23 +00004542/// If there are any memory operands, use OptimizeMemoryInst to sink their
4543/// address computing into the block when possible / profitable.
Sanjay Patelfc580a62015-09-21 23:03:16 +00004544bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
Evan Cheng1da25002008-02-26 02:42:37 +00004545 bool MadeChange = false;
Evan Cheng1da25002008-02-26 02:42:37 +00004546
Eric Christopher11e4df72015-02-26 22:38:43 +00004547 const TargetRegisterInfo *TRI =
Sanjay Patel4137d512017-06-07 14:29:52 +00004548 TM->getSubtargetImpl(*CS->getFunction())->getRegisterInfo();
Mehdi Amini8ac7a9d2015-07-07 19:07:19 +00004549 TargetLowering::AsmOperandInfoVector TargetConstraints =
4550 TLI->ParseConstraints(*DL, TRI, CS);
Dale Johannesenf95f59a2010-09-16 18:30:55 +00004551 unsigned ArgNo = 0;
John Thompson1094c802010-09-13 18:15:37 +00004552 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
4553 TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
Nadav Rotem465834c2012-07-24 10:51:42 +00004554
Evan Cheng1da25002008-02-26 02:42:37 +00004555 // Compute the constraint code and ConstraintType to use.
Dale Johannesence97d552010-06-25 21:55:36 +00004556 TLI->ComputeConstraintToUse(OpInfo, SDValue());
Evan Cheng1da25002008-02-26 02:42:37 +00004557
Eli Friedman666bbe32008-02-26 18:37:49 +00004558 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
4559 OpInfo.isIndirect) {
Chris Lattner7a277142011-01-15 07:14:54 +00004560 Value *OpVal = CS->getArgOperand(ArgNo++);
Sanjay Patelfc580a62015-09-21 23:03:16 +00004561 MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u);
Dale Johannesenf95f59a2010-09-16 18:30:55 +00004562 } else if (OpInfo.Type == InlineAsm::isInput)
4563 ArgNo++;
Evan Cheng1da25002008-02-26 02:42:37 +00004564 }
4565
4566 return MadeChange;
4567}
4568
Jun Bum Lim42301012017-03-17 19:05:21 +00004569/// \brief Check if all the uses of \p Val are equivalent (or free) zero or
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004570/// sign extensions.
Jun Bum Lim42301012017-03-17 19:05:21 +00004571static bool hasSameExtUse(Value *Val, const TargetLowering &TLI) {
4572 assert(!Val->use_empty() && "Input must have at least one use");
4573 const Instruction *FirstUser = cast<Instruction>(*Val->user_begin());
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004574 bool IsSExt = isa<SExtInst>(FirstUser);
4575 Type *ExtTy = FirstUser->getType();
Jun Bum Lim42301012017-03-17 19:05:21 +00004576 for (const User *U : Val->users()) {
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004577 const Instruction *UI = cast<Instruction>(U);
4578 if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI)))
4579 return false;
4580 Type *CurTy = UI->getType();
4581 // Same input and output types: Same instruction after CSE.
4582 if (CurTy == ExtTy)
4583 continue;
4584
4585 // If IsSExt is true, we are in this situation:
Jun Bum Lim42301012017-03-17 19:05:21 +00004586 // a = Val
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004587 // b = sext ty1 a to ty2
4588 // c = sext ty1 a to ty3
4589 // Assuming ty2 is shorter than ty3, this could be turned into:
Jun Bum Lim42301012017-03-17 19:05:21 +00004590 // a = Val
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004591 // b = sext ty1 a to ty2
4592 // c = sext ty2 b to ty3
4593 // However, the last sext is not free.
4594 if (IsSExt)
4595 return false;
4596
4597 // This is a ZExt, maybe this is free to extend from one type to another.
4598 // In that case, we would not account for a different use.
4599 Type *NarrowTy;
4600 Type *LargeTy;
4601 if (ExtTy->getScalarType()->getIntegerBitWidth() >
4602 CurTy->getScalarType()->getIntegerBitWidth()) {
4603 NarrowTy = CurTy;
4604 LargeTy = ExtTy;
4605 } else {
4606 NarrowTy = ExtTy;
4607 LargeTy = CurTy;
4608 }
4609
4610 if (!TLI.isZExtFree(NarrowTy, LargeTy))
4611 return false;
4612 }
4613 // All uses are the same or can be derived from one another for free.
4614 return true;
4615}
4616
Jun Bum Lim42301012017-03-17 19:05:21 +00004617/// \brief Try to speculatively promote extensions in \p Exts and continue
4618/// promoting through newly promoted operands recursively as far as doing so is
4619/// profitable. Save extensions profitably moved up, in \p ProfitablyMovedExts.
4620/// When some promotion happened, \p TPT contains the proper state to revert
4621/// them.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004622///
Jun Bum Lim42301012017-03-17 19:05:21 +00004623/// \return true if some promotion happened, false otherwise.
Jun Bum Lim42301012017-03-17 19:05:21 +00004624bool CodeGenPrepare::tryToPromoteExts(
4625 TypePromotionTransaction &TPT, const SmallVectorImpl<Instruction *> &Exts,
4626 SmallVectorImpl<Instruction *> &ProfitablyMovedExts,
4627 unsigned CreatedInstsCost) {
4628 bool Promoted = false;
4629
4630 // Iterate over all the extensions to try to promote them.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004631 for (auto I : Exts) {
Jun Bum Lim42301012017-03-17 19:05:21 +00004632 // Early check if we directly have ext(load).
4633 if (isa<LoadInst>(I->getOperand(0))) {
4634 ProfitablyMovedExts.push_back(I);
4635 continue;
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004636 }
Jun Bum Lim42301012017-03-17 19:05:21 +00004637
4638 // Check whether or not we want to do any promotion. The reason we have
4639 // this check inside the for loop is to catch the case where an extension
4640 // is directly fed by a load because in such case the extension can be moved
4641 // up without any promotion on its operands.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004642 if (!TLI || !TLI->enableExtLdPromotion() || DisableExtLdPromotion)
Jun Bum Lim42301012017-03-17 19:05:21 +00004643 return false;
4644
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004645 // Get the action to perform the promotion.
Jun Bum Lim42301012017-03-17 19:05:21 +00004646 TypePromotionHelper::Action TPH =
4647 TypePromotionHelper::getAction(I, InsertedInsts, *TLI, PromotedInsts);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004648 // Check if we can promote.
Jun Bum Lim42301012017-03-17 19:05:21 +00004649 if (!TPH) {
4650 // Save the current extension as we cannot move up through its operand.
4651 ProfitablyMovedExts.push_back(I);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004652 continue;
Jun Bum Lim42301012017-03-17 19:05:21 +00004653 }
4654
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004655 // Save the current state.
4656 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4657 TPT.getRestorationPoint();
4658 SmallVector<Instruction *, 4> NewExts;
Quentin Colombet1b274f92015-03-10 21:48:15 +00004659 unsigned NewCreatedInstsCost = 0;
4660 unsigned ExtCost = !TLI->isExtFree(I);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004661 // Promote.
Quentin Colombet1b274f92015-03-10 21:48:15 +00004662 Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost,
4663 &NewExts, nullptr, *TLI);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004664 assert(PromotedVal &&
4665 "TypePromotionHelper should have filtered out those cases");
4666
4667 // We would be able to merge only one extension in a load.
4668 // Therefore, if we have more than 1 new extension we heuristically
4669 // cut this search path, because it means we degrade the code quality.
4670 // With exactly 2, the transformation is neutral, because we will merge
4671 // one extension but leave one. However, we optimistically keep going,
4672 // because the new extension may be removed too.
Quentin Colombet1b274f92015-03-10 21:48:15 +00004673 long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost;
Jun Bum Limb99a06b2017-01-27 17:16:37 +00004674 // FIXME: It would be possible to propagate a negative value instead of
Jun Bum Lim42301012017-03-17 19:05:21 +00004675 // conservatively ceiling it to 0.
Jun Bum Limb99a06b2017-01-27 17:16:37 +00004676 TotalCreatedInstsCost =
4677 std::max((long long)0, (TotalCreatedInstsCost - ExtCost));
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004678 if (!StressExtLdPromotion &&
Quentin Colombet1b274f92015-03-10 21:48:15 +00004679 (TotalCreatedInstsCost > 1 ||
Mehdi Amini44ede332015-07-09 02:09:04 +00004680 !isPromotedInstructionLegal(*TLI, *DL, PromotedVal))) {
Jun Bum Lim42301012017-03-17 19:05:21 +00004681 // This promotion is not profitable, rollback to the previous state, and
4682 // save the current extension in ProfitablyMovedExts as the latest
4683 // speculative promotion turned out to be unprofitable.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004684 TPT.rollback(LastKnownGood);
Jun Bum Lim42301012017-03-17 19:05:21 +00004685 ProfitablyMovedExts.push_back(I);
4686 continue;
4687 }
4688 // Continue promoting NewExts as far as doing so is profitable.
4689 SmallVector<Instruction *, 2> NewlyMovedExts;
4690 (void)tryToPromoteExts(TPT, NewExts, NewlyMovedExts, TotalCreatedInstsCost);
4691 bool NewPromoted = false;
4692 for (auto ExtInst : NewlyMovedExts) {
4693 Instruction *MovedExt = cast<Instruction>(ExtInst);
4694 Value *ExtOperand = MovedExt->getOperand(0);
4695 // If we have reached to a load, we need this extra profitability check
4696 // as it could potentially be merged into an ext(load).
4697 if (isa<LoadInst>(ExtOperand) &&
4698 !(StressExtLdPromotion || NewCreatedInstsCost <= ExtCost ||
4699 (ExtOperand->hasOneUse() || hasSameExtUse(ExtOperand, *TLI))))
4700 continue;
4701
4702 ProfitablyMovedExts.push_back(MovedExt);
4703 NewPromoted = true;
4704 }
4705
4706 // If none of speculative promotions for NewExts is profitable, rollback
4707 // and save the current extension (I) as the last profitable extension.
4708 if (!NewPromoted) {
4709 TPT.rollback(LastKnownGood);
4710 ProfitablyMovedExts.push_back(I);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004711 continue;
4712 }
4713 // The promotion is profitable.
Jun Bum Lim42301012017-03-17 19:05:21 +00004714 Promoted = true;
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004715 }
Jun Bum Lim42301012017-03-17 19:05:21 +00004716 return Promoted;
4717}
4718
Jun Bum Limdee55652017-04-03 19:20:07 +00004719/// Merging redundant sexts when one is dominating the other.
4720bool CodeGenPrepare::mergeSExts(Function &F) {
4721 DominatorTree DT(F);
4722 bool Changed = false;
4723 for (auto &Entry : ValToSExtendedUses) {
4724 SExts &Insts = Entry.second;
4725 SExts CurPts;
4726 for (Instruction *Inst : Insts) {
4727 if (RemovedInsts.count(Inst) || !isa<SExtInst>(Inst) ||
4728 Inst->getOperand(0) != Entry.first)
4729 continue;
4730 bool inserted = false;
4731 for (auto &Pt : CurPts) {
4732 if (DT.dominates(Inst, Pt)) {
4733 Pt->replaceAllUsesWith(Inst);
4734 RemovedInsts.insert(Pt);
4735 Pt->removeFromParent();
4736 Pt = Inst;
4737 inserted = true;
4738 Changed = true;
4739 break;
4740 }
4741 if (!DT.dominates(Pt, Inst))
4742 // Give up if we need to merge in a common dominator as the
4743 // expermients show it is not profitable.
4744 continue;
4745 Inst->replaceAllUsesWith(Pt);
4746 RemovedInsts.insert(Inst);
4747 Inst->removeFromParent();
4748 inserted = true;
4749 Changed = true;
4750 break;
4751 }
4752 if (!inserted)
4753 CurPts.push_back(Inst);
4754 }
4755 }
4756 return Changed;
4757}
4758
Jun Bum Lim42301012017-03-17 19:05:21 +00004759/// Return true, if an ext(load) can be formed from an extension in
4760/// \p MovedExts.
4761bool CodeGenPrepare::canFormExtLd(
4762 const SmallVectorImpl<Instruction *> &MovedExts, LoadInst *&LI,
4763 Instruction *&Inst, bool HasPromoted) {
4764 for (auto *MovedExtInst : MovedExts) {
4765 if (isa<LoadInst>(MovedExtInst->getOperand(0))) {
4766 LI = cast<LoadInst>(MovedExtInst->getOperand(0));
4767 Inst = MovedExtInst;
4768 break;
4769 }
4770 }
4771 if (!LI)
4772 return false;
4773
4774 // If they're already in the same block, there's nothing to do.
4775 // Make the cheap checks first if we did not promote.
4776 // If we promoted, we need to check if it is indeed profitable.
4777 if (!HasPromoted && LI->getParent() == Inst->getParent())
4778 return false;
4779
Haicheng Wuabdef9e2017-07-15 02:12:16 +00004780 return TLI->isExtLoad(LI, Inst, *DL);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004781}
4782
Sanjay Patel4ac6b112015-09-21 22:47:23 +00004783/// Move a zext or sext fed by a load into the same basic block as the load,
4784/// unless conditions are unfavorable. This allows SelectionDAG to fold the
4785/// extend into the load.
Dan Gohman99429a02009-10-16 20:59:35 +00004786///
Jun Bum Limdee55652017-04-03 19:20:07 +00004787/// E.g.,
4788/// \code
4789/// %ld = load i32* %addr
4790/// %add = add nuw i32 %ld, 4
4791/// %zext = zext i32 %add to i64
4792// \endcode
4793/// =>
4794/// \code
4795/// %ld = load i32* %addr
4796/// %zext = zext i32 %ld to i64
4797/// %add = add nuw i64 %zext, 4
4798/// \encode
4799/// Note that the promotion in %add to i64 is done in tryToPromoteExts(), which
4800/// allow us to match zext(load i32*) to i64.
4801///
4802/// Also, try to promote the computations used to obtain a sign extended
4803/// value used into memory accesses.
4804/// E.g.,
4805/// \code
4806/// a = add nsw i32 b, 3
4807/// d = sext i32 a to i64
4808/// e = getelementptr ..., i64 d
4809/// \endcode
4810/// =>
4811/// \code
4812/// f = sext i32 b to i64
4813/// a = add nsw i64 f, 3
4814/// e = getelementptr ..., i64 a
4815/// \endcode
4816///
4817/// \p Inst[in/out] the extension may be modified during the process if some
4818/// promotions apply.
4819bool CodeGenPrepare::optimizeExt(Instruction *&Inst) {
4820 // ExtLoad formation and address type promotion infrastructure requires TLI to
4821 // be effective.
Chandler Carruth0f139b42016-11-04 06:54:00 +00004822 if (!TLI)
4823 return false;
4824
Jun Bum Limdee55652017-04-03 19:20:07 +00004825 bool AllowPromotionWithoutCommonHeader = false;
4826 /// See if it is an interesting sext operations for the address type
4827 /// promotion before trying to promote it, e.g., the ones with the right
4828 /// type and used in memory accesses.
4829 bool ATPConsiderable = TTI->shouldConsiderAddressTypePromotion(
4830 *Inst, AllowPromotionWithoutCommonHeader);
4831 TypePromotionTransaction TPT(RemovedInsts);
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004832 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
Jun Bum Lim42301012017-03-17 19:05:21 +00004833 TPT.getRestorationPoint();
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004834 SmallVector<Instruction *, 1> Exts;
Jun Bum Limdee55652017-04-03 19:20:07 +00004835 SmallVector<Instruction *, 2> SpeculativelyMovedExts;
4836 Exts.push_back(Inst);
Jun Bum Lim42301012017-03-17 19:05:21 +00004837
Jun Bum Limdee55652017-04-03 19:20:07 +00004838 bool HasPromoted = tryToPromoteExts(TPT, Exts, SpeculativelyMovedExts);
Jun Bum Lim42301012017-03-17 19:05:21 +00004839
Dan Gohman99429a02009-10-16 20:59:35 +00004840 // Look for a load being extended.
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004841 LoadInst *LI = nullptr;
Jun Bum Limdee55652017-04-03 19:20:07 +00004842 Instruction *ExtFedByLoad;
4843
4844 // Try to promote a chain of computation if it allows to form an extended
4845 // load.
4846 if (canFormExtLd(SpeculativelyMovedExts, LI, ExtFedByLoad, HasPromoted)) {
4847 assert(LI && ExtFedByLoad && "Expect a valid load and extension");
4848 TPT.commit();
4849 // Move the extend into the same block as the load
Sanjay Patel674d2c22017-08-29 14:07:48 +00004850 ExtFedByLoad->moveAfter(LI);
Jun Bum Limdee55652017-04-03 19:20:07 +00004851 // CGP does not check if the zext would be speculatively executed when moved
4852 // to the same basic block as the load. Preserving its original location
4853 // would pessimize the debugging experience, as well as negatively impact
4854 // the quality of sample pgo. We don't want to use "line 0" as that has a
4855 // size cost in the line-table section and logically the zext can be seen as
4856 // part of the load. Therefore we conservatively reuse the same debug
4857 // location for the load and the zext.
4858 ExtFedByLoad->setDebugLoc(LI->getDebugLoc());
4859 ++NumExtsMoved;
4860 Inst = ExtFedByLoad;
4861 return true;
4862 }
4863
4864 // Continue promoting SExts if known as considerable depending on targets.
4865 if (ATPConsiderable &&
4866 performAddressTypePromotion(Inst, AllowPromotionWithoutCommonHeader,
4867 HasPromoted, TPT, SpeculativelyMovedExts))
4868 return true;
4869
4870 TPT.rollback(LastKnownGood);
4871 return false;
4872}
4873
4874// Perform address type promotion if doing so is profitable.
4875// If AllowPromotionWithoutCommonHeader == false, we should find other sext
4876// instructions that sign extended the same initial value. However, if
4877// AllowPromotionWithoutCommonHeader == true, we expect promoting the
4878// extension is just profitable.
4879bool CodeGenPrepare::performAddressTypePromotion(
4880 Instruction *&Inst, bool AllowPromotionWithoutCommonHeader,
4881 bool HasPromoted, TypePromotionTransaction &TPT,
4882 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts) {
4883 bool Promoted = false;
4884 SmallPtrSet<Instruction *, 1> UnhandledExts;
4885 bool AllSeenFirst = true;
4886 for (auto I : SpeculativelyMovedExts) {
4887 Value *HeadOfChain = I->getOperand(0);
4888 DenseMap<Value *, Instruction *>::iterator AlreadySeen =
4889 SeenChainsForSExt.find(HeadOfChain);
4890 // If there is an unhandled SExt which has the same header, try to promote
4891 // it as well.
4892 if (AlreadySeen != SeenChainsForSExt.end()) {
4893 if (AlreadySeen->second != nullptr)
4894 UnhandledExts.insert(AlreadySeen->second);
4895 AllSeenFirst = false;
4896 }
4897 }
4898
4899 if (!AllSeenFirst || (AllowPromotionWithoutCommonHeader &&
4900 SpeculativelyMovedExts.size() == 1)) {
4901 TPT.commit();
4902 if (HasPromoted)
4903 Promoted = true;
4904 for (auto I : SpeculativelyMovedExts) {
4905 Value *HeadOfChain = I->getOperand(0);
4906 SeenChainsForSExt[HeadOfChain] = nullptr;
4907 ValToSExtendedUses[HeadOfChain].push_back(I);
4908 }
4909 // Update Inst as promotion happen.
4910 Inst = SpeculativelyMovedExts.pop_back_val();
4911 } else {
4912 // This is the first chain visited from the header, keep the current chain
4913 // as unhandled. Defer to promote this until we encounter another SExt
4914 // chain derived from the same header.
4915 for (auto I : SpeculativelyMovedExts) {
4916 Value *HeadOfChain = I->getOperand(0);
4917 SeenChainsForSExt[HeadOfChain] = Inst;
4918 }
Dan Gohman99429a02009-10-16 20:59:35 +00004919 return false;
Quentin Colombetfc2201e2014-12-17 01:36:17 +00004920 }
Dan Gohman99429a02009-10-16 20:59:35 +00004921
Jun Bum Limdee55652017-04-03 19:20:07 +00004922 if (!AllSeenFirst && !UnhandledExts.empty())
4923 for (auto VisitedSExt : UnhandledExts) {
4924 if (RemovedInsts.count(VisitedSExt))
4925 continue;
4926 TypePromotionTransaction TPT(RemovedInsts);
4927 SmallVector<Instruction *, 1> Exts;
4928 SmallVector<Instruction *, 2> Chains;
4929 Exts.push_back(VisitedSExt);
4930 bool HasPromoted = tryToPromoteExts(TPT, Exts, Chains);
4931 TPT.commit();
4932 if (HasPromoted)
4933 Promoted = true;
4934 for (auto I : Chains) {
4935 Value *HeadOfChain = I->getOperand(0);
4936 // Mark this as handled.
4937 SeenChainsForSExt[HeadOfChain] = nullptr;
4938 ValToSExtendedUses[HeadOfChain].push_back(I);
4939 }
4940 }
4941 return Promoted;
Dan Gohman99429a02009-10-16 20:59:35 +00004942}
4943
Sanjay Patelfc580a62015-09-21 23:03:16 +00004944bool CodeGenPrepare::optimizeExtUses(Instruction *I) {
Evan Chengd3d80172007-12-05 23:58:20 +00004945 BasicBlock *DefBB = I->getParent();
4946
Bob Wilsonff714f92010-09-21 21:44:14 +00004947 // If the result of a {s|z}ext and its source are both live out, rewrite all
Evan Chengd3d80172007-12-05 23:58:20 +00004948 // other uses of the source with result of extension.
4949 Value *Src = I->getOperand(0);
4950 if (Src->hasOneUse())
4951 return false;
4952
Evan Cheng2011df42007-12-13 07:50:36 +00004953 // Only do this xform if truncating is free.
Gabor Greifaa261722008-02-26 19:13:21 +00004954 if (TLI && !TLI->isTruncateFree(I->getType(), Src->getType()))
Evan Cheng37c36ed2007-12-13 03:32:53 +00004955 return false;
4956
Evan Cheng7bc89422007-12-12 00:51:06 +00004957 // Only safe to perform the optimization if the source is also defined in
Evan Cheng63d33cf2007-12-12 02:53:41 +00004958 // this block.
4959 if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent())
Evan Cheng7bc89422007-12-12 00:51:06 +00004960 return false;
4961
Evan Chengd3d80172007-12-05 23:58:20 +00004962 bool DefIsLiveOut = false;
Chandler Carruthcdf47882014-03-09 03:16:01 +00004963 for (User *U : I->users()) {
4964 Instruction *UI = cast<Instruction>(U);
Evan Chengd3d80172007-12-05 23:58:20 +00004965
4966 // Figure out which BB this ext is used in.
Chandler Carruthcdf47882014-03-09 03:16:01 +00004967 BasicBlock *UserBB = UI->getParent();
Evan Chengd3d80172007-12-05 23:58:20 +00004968 if (UserBB == DefBB) continue;
4969 DefIsLiveOut = true;
4970 break;
4971 }
4972 if (!DefIsLiveOut)
4973 return false;
4974
Jim Grosbach0f38c1e2013-04-15 17:40:48 +00004975 // Make sure none of the uses are PHI nodes.
Chandler Carruthcdf47882014-03-09 03:16:01 +00004976 for (User *U : Src->users()) {
4977 Instruction *UI = cast<Instruction>(U);
4978 BasicBlock *UserBB = UI->getParent();
Evan Cheng37c36ed2007-12-13 03:32:53 +00004979 if (UserBB == DefBB) continue;
4980 // Be conservative. We don't want this xform to end up introducing
4981 // reloads just before load / store instructions.
Chandler Carruthcdf47882014-03-09 03:16:01 +00004982 if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI))
Evan Cheng63d33cf2007-12-12 02:53:41 +00004983 return false;
4984 }
4985
Evan Chengd3d80172007-12-05 23:58:20 +00004986 // InsertedTruncs - Only insert one trunc in each block once.
4987 DenseMap<BasicBlock*, Instruction*> InsertedTruncs;
4988
4989 bool MadeChange = false;
Chandler Carruthcdf47882014-03-09 03:16:01 +00004990 for (Use &U : Src->uses()) {
4991 Instruction *User = cast<Instruction>(U.getUser());
Evan Chengd3d80172007-12-05 23:58:20 +00004992
4993 // Figure out which BB this ext is used in.
4994 BasicBlock *UserBB = User->getParent();
4995 if (UserBB == DefBB) continue;
4996
4997 // Both src and def are live in this block. Rewrite the use.
4998 Instruction *&InsertedTrunc = InsertedTruncs[UserBB];
4999
5000 if (!InsertedTrunc) {
Bill Wendling8ddfc092011-08-16 20:45:24 +00005001 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00005002 assert(InsertPt != UserBB->end());
5003 InsertedTrunc = new TruncInst(I, Src->getType(), "", &*InsertPt);
Ahmed Bougachaf3299142015-06-17 20:44:32 +00005004 InsertedInsts.insert(InsertedTrunc);
Evan Chengd3d80172007-12-05 23:58:20 +00005005 }
5006
5007 // Replace a use of the {s|z}ext source with a use of the result.
Chandler Carruthcdf47882014-03-09 03:16:01 +00005008 U = InsertedTrunc;
Cameron Zwarichced753f2011-01-05 17:27:27 +00005009 ++NumExtUses;
Evan Chengd3d80172007-12-05 23:58:20 +00005010 MadeChange = true;
5011 }
5012
5013 return MadeChange;
5014}
5015
Geoff Berry5256fca2015-11-20 22:34:39 +00005016// Find loads whose uses only use some of the loaded value's bits. Add an "and"
5017// just after the load if the target can fold this into one extload instruction,
5018// with the hope of eliminating some of the other later "and" instructions using
5019// the loaded value. "and"s that are made trivially redundant by the insertion
5020// of the new "and" are removed by this function, while others (e.g. those whose
5021// path from the load goes through a phi) are left for isel to potentially
5022// remove.
5023//
5024// For example:
5025//
5026// b0:
5027// x = load i32
5028// ...
5029// b1:
5030// y = and x, 0xff
5031// z = use y
5032//
5033// becomes:
5034//
5035// b0:
5036// x = load i32
5037// x' = and x, 0xff
5038// ...
5039// b1:
5040// z = use x'
5041//
5042// whereas:
5043//
5044// b0:
5045// x1 = load i32
5046// ...
5047// b1:
5048// x2 = load i32
5049// ...
5050// b2:
5051// x = phi x1, x2
5052// y = and x, 0xff
5053//
5054// becomes (after a call to optimizeLoadExt for each load):
5055//
5056// b0:
5057// x1 = load i32
5058// x1' = and x1, 0xff
5059// ...
5060// b1:
5061// x2 = load i32
5062// x2' = and x2, 0xff
5063// ...
5064// b2:
5065// x = phi x1', x2'
5066// y = and x, 0xff
Geoff Berry5256fca2015-11-20 22:34:39 +00005067bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
Geoff Berry5256fca2015-11-20 22:34:39 +00005068 if (!Load->isSimple() ||
5069 !(Load->getType()->isIntegerTy() || Load->getType()->isPointerTy()))
5070 return false;
5071
Geoff Berry5d534b62017-02-21 18:53:14 +00005072 // Skip loads we've already transformed.
5073 if (Load->hasOneUse() &&
5074 InsertedInsts.count(cast<Instruction>(*Load->user_begin())))
5075 return false;
Geoff Berry5256fca2015-11-20 22:34:39 +00005076
5077 // Look at all uses of Load, looking through phis, to determine how many bits
5078 // of the loaded value are needed.
5079 SmallVector<Instruction *, 8> WorkList;
5080 SmallPtrSet<Instruction *, 16> Visited;
5081 SmallVector<Instruction *, 8> AndsToMaybeRemove;
5082 for (auto *U : Load->users())
5083 WorkList.push_back(cast<Instruction>(U));
5084
5085 EVT LoadResultVT = TLI->getValueType(*DL, Load->getType());
5086 unsigned BitWidth = LoadResultVT.getSizeInBits();
5087 APInt DemandBits(BitWidth, 0);
5088 APInt WidestAndBits(BitWidth, 0);
5089
5090 while (!WorkList.empty()) {
5091 Instruction *I = WorkList.back();
5092 WorkList.pop_back();
5093
5094 // Break use-def graph loops.
5095 if (!Visited.insert(I).second)
5096 continue;
5097
5098 // For a PHI node, push all of its users.
5099 if (auto *Phi = dyn_cast<PHINode>(I)) {
5100 for (auto *U : Phi->users())
5101 WorkList.push_back(cast<Instruction>(U));
5102 continue;
5103 }
5104
5105 switch (I->getOpcode()) {
Eugene Zelenko900b6332017-08-29 22:32:07 +00005106 case Instruction::And: {
Geoff Berry5256fca2015-11-20 22:34:39 +00005107 auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1));
5108 if (!AndC)
5109 return false;
5110 APInt AndBits = AndC->getValue();
5111 DemandBits |= AndBits;
5112 // Keep track of the widest and mask we see.
5113 if (AndBits.ugt(WidestAndBits))
5114 WidestAndBits = AndBits;
5115 if (AndBits == WidestAndBits && I->getOperand(0) == Load)
5116 AndsToMaybeRemove.push_back(I);
5117 break;
5118 }
5119
Eugene Zelenko900b6332017-08-29 22:32:07 +00005120 case Instruction::Shl: {
Geoff Berry5256fca2015-11-20 22:34:39 +00005121 auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1));
5122 if (!ShlC)
5123 return false;
5124 uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1);
Craig Topperfc947bc2017-04-18 17:14:21 +00005125 DemandBits.setLowBits(BitWidth - ShiftAmt);
Geoff Berry5256fca2015-11-20 22:34:39 +00005126 break;
5127 }
5128
Eugene Zelenko900b6332017-08-29 22:32:07 +00005129 case Instruction::Trunc: {
Geoff Berry5256fca2015-11-20 22:34:39 +00005130 EVT TruncVT = TLI->getValueType(*DL, I->getType());
5131 unsigned TruncBitWidth = TruncVT.getSizeInBits();
Craig Topperfc947bc2017-04-18 17:14:21 +00005132 DemandBits.setLowBits(TruncBitWidth);
Geoff Berry5256fca2015-11-20 22:34:39 +00005133 break;
5134 }
5135
5136 default:
5137 return false;
5138 }
5139 }
5140
5141 uint32_t ActiveBits = DemandBits.getActiveBits();
5142 // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the
5143 // target even if isLoadExtLegal says an i1 EXTLOAD is valid. For example,
5144 // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but
5145 // (and (load x) 1) is not matched as a single instruction, rather as a LDR
5146 // followed by an AND.
5147 // TODO: Look into removing this restriction by fixing backends to either
5148 // return false for isLoadExtLegal for i1 or have them select this pattern to
5149 // a single instruction.
5150 //
5151 // Also avoid hoisting if we didn't see any ands with the exact DemandBits
5152 // mask, since these are the only ands that will be removed by isel.
Craig Topperd33ee1b2017-04-03 16:34:59 +00005153 if (ActiveBits <= 1 || !DemandBits.isMask(ActiveBits) ||
Geoff Berry5256fca2015-11-20 22:34:39 +00005154 WidestAndBits != DemandBits)
5155 return false;
5156
5157 LLVMContext &Ctx = Load->getType()->getContext();
5158 Type *TruncTy = Type::getIntNTy(Ctx, ActiveBits);
5159 EVT TruncVT = TLI->getValueType(*DL, TruncTy);
5160
5161 // Reject cases that won't be matched as extloads.
5162 if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() ||
5163 !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT))
5164 return false;
5165
5166 IRBuilder<> Builder(Load->getNextNode());
5167 auto *NewAnd = dyn_cast<Instruction>(
5168 Builder.CreateAnd(Load, ConstantInt::get(Ctx, DemandBits)));
Geoff Berry5d534b62017-02-21 18:53:14 +00005169 // Mark this instruction as "inserted by CGP", so that other
5170 // optimizations don't touch it.
5171 InsertedInsts.insert(NewAnd);
Geoff Berry5256fca2015-11-20 22:34:39 +00005172
5173 // Replace all uses of load with new and (except for the use of load in the
5174 // new and itself).
5175 Load->replaceAllUsesWith(NewAnd);
5176 NewAnd->setOperand(0, Load);
5177
5178 // Remove any and instructions that are now redundant.
5179 for (auto *And : AndsToMaybeRemove)
5180 // Check that the and mask is the same as the one we decided to put on the
5181 // new and.
5182 if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) {
5183 And->replaceAllUsesWith(NewAnd);
5184 if (&*CurInstIterator == And)
5185 CurInstIterator = std::next(And->getIterator());
5186 And->eraseFromParent();
5187 ++NumAndUses;
5188 }
5189
5190 ++NumAndsAdded;
5191 return true;
5192}
5193
Sanjay Patel69a50a12015-10-19 21:59:12 +00005194/// Check if V (an operand of a select instruction) is an expensive instruction
5195/// that is only used once.
5196static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V) {
5197 auto *I = dyn_cast<Instruction>(V);
5198 // If it's safe to speculatively execute, then it should not have side
5199 // effects; therefore, it's safe to sink and possibly *not* execute.
Rafael Espindola84921b92015-10-24 23:11:13 +00005200 return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) &&
5201 TTI->getUserCost(I) >= TargetTransformInfo::TCC_Expensive;
Sanjay Patel69a50a12015-10-19 21:59:12 +00005202}
5203
Sanjay Patel4ac6b112015-09-21 22:47:23 +00005204/// Returns true if a SelectInst should be turned into an explicit branch.
Sanjay Patel69a50a12015-10-19 21:59:12 +00005205static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
Sanjay Pateld66607b2016-04-26 17:11:17 +00005206 const TargetLowering *TLI,
Sanjay Patel69a50a12015-10-19 21:59:12 +00005207 SelectInst *SI) {
Sanjay Pateld66607b2016-04-26 17:11:17 +00005208 // If even a predictable select is cheap, then a branch can't be cheaper.
5209 if (!TLI->isPredictableSelectExpensive())
5210 return false;
5211
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005212 // FIXME: This should use the same heuristics as IfConversion to determine
Sanjay Pateld66607b2016-04-26 17:11:17 +00005213 // whether a select is better represented as a branch.
5214
5215 // If metadata tells us that the select condition is obviously predictable,
5216 // then we want to replace the select with a branch.
5217 uint64_t TrueWeight, FalseWeight;
5218 if (SI->extractProfMetadata(TrueWeight, FalseWeight)) {
5219 uint64_t Max = std::max(TrueWeight, FalseWeight);
5220 uint64_t Sum = TrueWeight + FalseWeight;
Sanjay Patelc7b91e62016-05-09 17:31:55 +00005221 if (Sum != 0) {
5222 auto Probability = BranchProbability::getBranchProbability(Max, Sum);
5223 if (Probability > TLI->getPredictableBranchThreshold())
5224 return true;
5225 }
Sanjay Pateld66607b2016-04-26 17:11:17 +00005226 }
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005227
5228 CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition());
5229
Sanjay Patel4e652762015-09-28 22:14:51 +00005230 // If a branch is predictable, an out-of-order CPU can avoid blocking on its
5231 // comparison condition. If the compare has more than one use, there's
5232 // probably another cmov or setcc around, so it's not worth emitting a branch.
Sanjay Patel5e5f0e92015-09-28 21:44:46 +00005233 if (!Cmp || !Cmp->hasOneUse())
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005234 return false;
5235
Sanjay Patel69a50a12015-10-19 21:59:12 +00005236 // If either operand of the select is expensive and only needed on one side
5237 // of the select, we should form a branch.
5238 if (sinkSelectOperand(TTI, SI->getTrueValue()) ||
5239 sinkSelectOperand(TTI, SI->getFalseValue()))
5240 return true;
5241
5242 return false;
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005243}
5244
Dehao Chen9bbb9412016-09-12 20:23:28 +00005245/// If \p isTrue is true, return the true value of \p SI, otherwise return
5246/// false value of \p SI. If the true/false value of \p SI is defined by any
5247/// select instructions in \p Selects, look through the defining select
5248/// instruction until the true/false value is not defined in \p Selects.
5249static Value *getTrueOrFalseValue(
5250 SelectInst *SI, bool isTrue,
5251 const SmallPtrSet<const Instruction *, 2> &Selects) {
5252 Value *V;
5253
5254 for (SelectInst *DefSI = SI; DefSI != nullptr && Selects.count(DefSI);
5255 DefSI = dyn_cast<SelectInst>(V)) {
Dehao Chenc32d7122016-09-12 20:29:54 +00005256 assert(DefSI->getCondition() == SI->getCondition() &&
Dehao Chen9bbb9412016-09-12 20:23:28 +00005257 "The condition of DefSI does not match with SI");
5258 V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue());
5259 }
5260 return V;
5261}
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005262
Nadav Rotem9d832022012-09-02 12:10:19 +00005263/// If we have a SelectInst that will likely profit from branch prediction,
5264/// turn it into a branch.
Sanjay Patelfc580a62015-09-21 23:03:16 +00005265bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
Dehao Chen9bbb9412016-09-12 20:23:28 +00005266 // Find all consecutive select instructions that share the same condition.
5267 SmallVector<SelectInst *, 2> ASI;
5268 ASI.push_back(SI);
5269 for (BasicBlock::iterator It = ++BasicBlock::iterator(SI);
5270 It != SI->getParent()->end(); ++It) {
5271 SelectInst *I = dyn_cast<SelectInst>(&*It);
5272 if (I && SI->getCondition() == I->getCondition()) {
5273 ASI.push_back(I);
5274 } else {
5275 break;
5276 }
5277 }
5278
5279 SelectInst *LastSI = ASI.back();
5280 // Increment the current iterator to skip all the rest of select instructions
5281 // because they will be either "not lowered" or "all lowered" to branch.
5282 CurInstIterator = std::next(LastSI->getIterator());
5283
Nadav Rotem9d832022012-09-02 12:10:19 +00005284 bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1);
5285
5286 // Can we convert the 'select' to CF ?
Sanjay Patela31b0c02016-04-26 00:47:39 +00005287 if (DisableSelectToBranch || OptSize || !TLI || VectorCond ||
5288 SI->getMetadata(LLVMContext::MD_unpredictable))
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005289 return false;
5290
Nadav Rotem9d832022012-09-02 12:10:19 +00005291 TargetLowering::SelectSupportKind SelectKind;
5292 if (VectorCond)
5293 SelectKind = TargetLowering::VectorMaskSelect;
5294 else if (SI->getType()->isVectorTy())
5295 SelectKind = TargetLowering::ScalarCondVectorVal;
5296 else
5297 SelectKind = TargetLowering::ScalarValSelect;
5298
Sanjay Pateld66607b2016-04-26 17:11:17 +00005299 if (TLI->isSelectSupported(SelectKind) &&
5300 !isFormingBranchFromSelectProfitable(TTI, TLI, SI))
5301 return false;
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005302
5303 ModifiedDT = true;
5304
Sanjay Patel69a50a12015-10-19 21:59:12 +00005305 // Transform a sequence like this:
5306 // start:
5307 // %cmp = cmp uge i32 %a, %b
5308 // %sel = select i1 %cmp, i32 %c, i32 %d
5309 //
5310 // Into:
5311 // start:
5312 // %cmp = cmp uge i32 %a, %b
5313 // br i1 %cmp, label %select.true, label %select.false
5314 // select.true:
5315 // br label %select.end
5316 // select.false:
5317 // br label %select.end
5318 // select.end:
5319 // %sel = phi i32 [ %c, %select.true ], [ %d, %select.false ]
5320 //
5321 // In addition, we may sink instructions that produce %c or %d from
5322 // the entry block into the destination(s) of the new branch.
5323 // If the true or false blocks do not contain a sunken instruction, that
5324 // block and its branch may be optimized away. In that case, one side of the
5325 // first branch will point directly to select.end, and the corresponding PHI
5326 // predecessor block will be the start block.
5327
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005328 // First, we split the block containing the select into 2 blocks.
5329 BasicBlock *StartBlock = SI->getParent();
Dehao Chen9bbb9412016-09-12 20:23:28 +00005330 BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(LastSI));
Sanjay Patel69a50a12015-10-19 21:59:12 +00005331 BasicBlock *EndBlock = StartBlock->splitBasicBlock(SplitPt, "select.end");
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005332
Sanjay Patel69a50a12015-10-19 21:59:12 +00005333 // Delete the unconditional branch that was just created by the split.
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005334 StartBlock->getTerminator()->eraseFromParent();
Sanjay Patel69a50a12015-10-19 21:59:12 +00005335
5336 // These are the new basic blocks for the conditional branch.
5337 // At least one will become an actual new basic block.
5338 BasicBlock *TrueBlock = nullptr;
5339 BasicBlock *FalseBlock = nullptr;
Dehao Chen9bbb9412016-09-12 20:23:28 +00005340 BranchInst *TrueBranch = nullptr;
5341 BranchInst *FalseBranch = nullptr;
Sanjay Patel69a50a12015-10-19 21:59:12 +00005342
5343 // Sink expensive instructions into the conditional blocks to avoid executing
5344 // them speculatively.
Dehao Chen9bbb9412016-09-12 20:23:28 +00005345 for (SelectInst *SI : ASI) {
5346 if (sinkSelectOperand(TTI, SI->getTrueValue())) {
5347 if (TrueBlock == nullptr) {
5348 TrueBlock = BasicBlock::Create(SI->getContext(), "select.true.sink",
5349 EndBlock->getParent(), EndBlock);
5350 TrueBranch = BranchInst::Create(EndBlock, TrueBlock);
5351 }
5352 auto *TrueInst = cast<Instruction>(SI->getTrueValue());
5353 TrueInst->moveBefore(TrueBranch);
5354 }
5355 if (sinkSelectOperand(TTI, SI->getFalseValue())) {
5356 if (FalseBlock == nullptr) {
5357 FalseBlock = BasicBlock::Create(SI->getContext(), "select.false.sink",
5358 EndBlock->getParent(), EndBlock);
5359 FalseBranch = BranchInst::Create(EndBlock, FalseBlock);
5360 }
5361 auto *FalseInst = cast<Instruction>(SI->getFalseValue());
5362 FalseInst->moveBefore(FalseBranch);
5363 }
Sanjay Patel69a50a12015-10-19 21:59:12 +00005364 }
5365
5366 // If there was nothing to sink, then arbitrarily choose the 'false' side
5367 // for a new input value to the PHI.
5368 if (TrueBlock == FalseBlock) {
5369 assert(TrueBlock == nullptr &&
5370 "Unexpected basic block transform while optimizing select");
5371
5372 FalseBlock = BasicBlock::Create(SI->getContext(), "select.false",
5373 EndBlock->getParent(), EndBlock);
5374 BranchInst::Create(EndBlock, FalseBlock);
5375 }
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005376
5377 // Insert the real conditional branch based on the original condition.
Sanjay Patel69a50a12015-10-19 21:59:12 +00005378 // If we did not create a new block for one of the 'true' or 'false' paths
5379 // of the condition, it means that side of the branch goes to the end block
5380 // directly and the path originates from the start block from the point of
5381 // view of the new PHI.
Xinliang David Li241e6c72016-09-03 21:26:36 +00005382 BasicBlock *TT, *FT;
Sanjay Patel69a50a12015-10-19 21:59:12 +00005383 if (TrueBlock == nullptr) {
Xinliang David Li241e6c72016-09-03 21:26:36 +00005384 TT = EndBlock;
5385 FT = FalseBlock;
Sanjay Patel69a50a12015-10-19 21:59:12 +00005386 TrueBlock = StartBlock;
5387 } else if (FalseBlock == nullptr) {
Xinliang David Li241e6c72016-09-03 21:26:36 +00005388 TT = TrueBlock;
5389 FT = EndBlock;
Sanjay Patel69a50a12015-10-19 21:59:12 +00005390 FalseBlock = StartBlock;
5391 } else {
Xinliang David Li241e6c72016-09-03 21:26:36 +00005392 TT = TrueBlock;
5393 FT = FalseBlock;
Sanjay Patel69a50a12015-10-19 21:59:12 +00005394 }
Xinliang David Li241e6c72016-09-03 21:26:36 +00005395 IRBuilder<>(SI).CreateCondBr(SI->getCondition(), TT, FT, SI);
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005396
Dehao Chen9bbb9412016-09-12 20:23:28 +00005397 SmallPtrSet<const Instruction *, 2> INS;
5398 INS.insert(ASI.begin(), ASI.end());
5399 // Use reverse iterator because later select may use the value of the
5400 // earlier select, and we need to propagate value through earlier select
5401 // to get the PHI operand.
5402 for (auto It = ASI.rbegin(); It != ASI.rend(); ++It) {
5403 SelectInst *SI = *It;
5404 // The select itself is replaced with a PHI Node.
5405 PHINode *PN = PHINode::Create(SI->getType(), 2, "", &EndBlock->front());
5406 PN->takeName(SI);
5407 PN->addIncoming(getTrueOrFalseValue(SI, true, INS), TrueBlock);
5408 PN->addIncoming(getTrueOrFalseValue(SI, false, INS), FalseBlock);
Sanjay Patel69a50a12015-10-19 21:59:12 +00005409
Dehao Chen9bbb9412016-09-12 20:23:28 +00005410 SI->replaceAllUsesWith(PN);
5411 SI->eraseFromParent();
5412 INS.erase(SI);
5413 ++NumSelectsExpanded;
5414 }
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005415
5416 // Instruct OptimizeBlock to skip to the next block.
5417 CurInstIterator = StartBlock->end();
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00005418 return true;
5419}
5420
Benjamin Kramer573ff362014-03-01 17:24:40 +00005421static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
Tim Northoveraeb8e062014-02-19 10:02:43 +00005422 SmallVector<int, 16> Mask(SVI->getShuffleMask());
5423 int SplatElem = -1;
5424 for (unsigned i = 0; i < Mask.size(); ++i) {
5425 if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
5426 return false;
5427 SplatElem = Mask[i];
5428 }
5429
5430 return true;
5431}
5432
5433/// Some targets have expensive vector shifts if the lanes aren't all the same
5434/// (e.g. x86 only introduced "vpsllvd" and friends with AVX2). In these cases
5435/// it's often worth sinking a shufflevector splat down to its use so that
5436/// codegen can spot all lanes are identical.
Sanjay Patelfc580a62015-09-21 23:03:16 +00005437bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
Tim Northoveraeb8e062014-02-19 10:02:43 +00005438 BasicBlock *DefBB = SVI->getParent();
5439
5440 // Only do this xform if variable vector shifts are particularly expensive.
5441 if (!TLI || !TLI->isVectorShiftByScalarCheap(SVI->getType()))
5442 return false;
5443
5444 // We only expect better codegen by sinking a shuffle if we can recognise a
5445 // constant splat.
5446 if (!isBroadcastShuffle(SVI))
5447 return false;
5448
5449 // InsertedShuffles - Only insert a shuffle in each block once.
5450 DenseMap<BasicBlock*, Instruction*> InsertedShuffles;
5451
5452 bool MadeChange = false;
Chandler Carruthcdf47882014-03-09 03:16:01 +00005453 for (User *U : SVI->users()) {
5454 Instruction *UI = cast<Instruction>(U);
Tim Northoveraeb8e062014-02-19 10:02:43 +00005455
5456 // Figure out which BB this ext is used in.
Chandler Carruthcdf47882014-03-09 03:16:01 +00005457 BasicBlock *UserBB = UI->getParent();
Tim Northoveraeb8e062014-02-19 10:02:43 +00005458 if (UserBB == DefBB) continue;
5459
5460 // For now only apply this when the splat is used by a shift instruction.
Chandler Carruthcdf47882014-03-09 03:16:01 +00005461 if (!UI->isShift()) continue;
Tim Northoveraeb8e062014-02-19 10:02:43 +00005462
5463 // Everything checks out, sink the shuffle if the user's block doesn't
5464 // already have a copy.
5465 Instruction *&InsertedShuffle = InsertedShuffles[UserBB];
5466
5467 if (!InsertedShuffle) {
5468 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00005469 assert(InsertPt != UserBB->end());
5470 InsertedShuffle =
5471 new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
5472 SVI->getOperand(2), "", &*InsertPt);
Tim Northoveraeb8e062014-02-19 10:02:43 +00005473 }
5474
Chandler Carruthcdf47882014-03-09 03:16:01 +00005475 UI->replaceUsesOfWith(SVI, InsertedShuffle);
Tim Northoveraeb8e062014-02-19 10:02:43 +00005476 MadeChange = true;
5477 }
5478
5479 // If we removed all uses, nuke the shuffle.
5480 if (SVI->use_empty()) {
5481 SVI->eraseFromParent();
5482 MadeChange = true;
5483 }
5484
5485 return MadeChange;
5486}
5487
Sanjay Patel0ed9aea2015-11-02 23:22:49 +00005488bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) {
5489 if (!TLI || !DL)
5490 return false;
5491
5492 Value *Cond = SI->getCondition();
5493 Type *OldType = Cond->getType();
5494 LLVMContext &Context = Cond->getContext();
5495 MVT RegType = TLI->getRegisterType(Context, TLI->getValueType(*DL, OldType));
5496 unsigned RegWidth = RegType.getSizeInBits();
5497
5498 if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth())
5499 return false;
5500
5501 // If the register width is greater than the type width, expand the condition
5502 // of the switch instruction and each case constant to the width of the
5503 // register. By widening the type of the switch condition, subsequent
5504 // comparisons (for case comparisons) will not need to be extended to the
5505 // preferred register width, so we will potentially eliminate N-1 extends,
5506 // where N is the number of cases in the switch.
5507 auto *NewType = Type::getIntNTy(Context, RegWidth);
5508
5509 // Zero-extend the switch condition and case constants unless the switch
5510 // condition is a function argument that is already being sign-extended.
5511 // In that case, we can avoid an unnecessary mask/extension by sign-extending
5512 // everything instead.
5513 Instruction::CastOps ExtType = Instruction::ZExt;
5514 if (auto *Arg = dyn_cast<Argument>(Cond))
5515 if (Arg->hasSExtAttr())
5516 ExtType = Instruction::SExt;
5517
5518 auto *ExtInst = CastInst::Create(ExtType, Cond, NewType);
5519 ExtInst->insertBefore(SI);
5520 SI->setCondition(ExtInst);
Chandler Carruth927d8e62017-04-12 07:27:28 +00005521 for (auto Case : SI->cases()) {
Sanjay Patel0ed9aea2015-11-02 23:22:49 +00005522 APInt NarrowConst = Case.getCaseValue()->getValue();
5523 APInt WideConst = (ExtType == Instruction::ZExt) ?
5524 NarrowConst.zext(RegWidth) : NarrowConst.sext(RegWidth);
5525 Case.setValue(ConstantInt::get(Context, WideConst));
5526 }
5527
5528 return true;
5529}
5530
Zaara Syeda3a7578c2017-05-31 17:12:38 +00005531
Quentin Colombetc32615d2014-10-31 17:52:53 +00005532namespace {
Eugene Zelenko900b6332017-08-29 22:32:07 +00005533
Quentin Colombetc32615d2014-10-31 17:52:53 +00005534/// \brief Helper class to promote a scalar operation to a vector one.
5535/// This class is used to move downward extractelement transition.
5536/// E.g.,
5537/// a = vector_op <2 x i32>
5538/// b = extractelement <2 x i32> a, i32 0
5539/// c = scalar_op b
5540/// store c
5541///
5542/// =>
5543/// a = vector_op <2 x i32>
5544/// c = vector_op a (equivalent to scalar_op on the related lane)
5545/// * d = extractelement <2 x i32> c, i32 0
5546/// * store d
5547/// Assuming both extractelement and store can be combine, we get rid of the
5548/// transition.
5549class VectorPromoteHelper {
Mehdi Amini44ede332015-07-09 02:09:04 +00005550 /// DataLayout associated with the current module.
5551 const DataLayout &DL;
5552
Quentin Colombetc32615d2014-10-31 17:52:53 +00005553 /// Used to perform some checks on the legality of vector operations.
5554 const TargetLowering &TLI;
5555
5556 /// Used to estimated the cost of the promoted chain.
5557 const TargetTransformInfo &TTI;
5558
5559 /// The transition being moved downwards.
5560 Instruction *Transition;
Eugene Zelenko900b6332017-08-29 22:32:07 +00005561
Quentin Colombetc32615d2014-10-31 17:52:53 +00005562 /// The sequence of instructions to be promoted.
5563 SmallVector<Instruction *, 4> InstsToBePromoted;
Eugene Zelenko900b6332017-08-29 22:32:07 +00005564
Quentin Colombetc32615d2014-10-31 17:52:53 +00005565 /// Cost of combining a store and an extract.
5566 unsigned StoreExtractCombineCost;
Eugene Zelenko900b6332017-08-29 22:32:07 +00005567
Quentin Colombetc32615d2014-10-31 17:52:53 +00005568 /// Instruction that will be combined with the transition.
Eugene Zelenko900b6332017-08-29 22:32:07 +00005569 Instruction *CombineInst = nullptr;
Quentin Colombetc32615d2014-10-31 17:52:53 +00005570
5571 /// \brief The instruction that represents the current end of the transition.
5572 /// Since we are faking the promotion until we reach the end of the chain
5573 /// of computation, we need a way to get the current end of the transition.
5574 Instruction *getEndOfTransition() const {
5575 if (InstsToBePromoted.empty())
5576 return Transition;
5577 return InstsToBePromoted.back();
5578 }
5579
5580 /// \brief Return the index of the original value in the transition.
5581 /// E.g., for "extractelement <2 x i32> c, i32 1" the original value,
5582 /// c, is at index 0.
5583 unsigned getTransitionOriginalValueIdx() const {
5584 assert(isa<ExtractElementInst>(Transition) &&
5585 "Other kind of transitions are not supported yet");
5586 return 0;
5587 }
5588
5589 /// \brief Return the index of the index in the transition.
5590 /// E.g., for "extractelement <2 x i32> c, i32 0" the index
5591 /// is at index 1.
5592 unsigned getTransitionIdx() const {
5593 assert(isa<ExtractElementInst>(Transition) &&
5594 "Other kind of transitions are not supported yet");
5595 return 1;
5596 }
5597
5598 /// \brief Get the type of the transition.
5599 /// This is the type of the original value.
5600 /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the
5601 /// transition is <2 x i32>.
5602 Type *getTransitionType() const {
5603 return Transition->getOperand(getTransitionOriginalValueIdx())->getType();
5604 }
5605
5606 /// \brief Promote \p ToBePromoted by moving \p Def downward through.
5607 /// I.e., we have the following sequence:
5608 /// Def = Transition <ty1> a to <ty2>
5609 /// b = ToBePromoted <ty2> Def, ...
5610 /// =>
5611 /// b = ToBePromoted <ty1> a, ...
5612 /// Def = Transition <ty1> ToBePromoted to <ty2>
5613 void promoteImpl(Instruction *ToBePromoted);
5614
5615 /// \brief Check whether or not it is profitable to promote all the
5616 /// instructions enqueued to be promoted.
5617 bool isProfitableToPromote() {
5618 Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx());
5619 unsigned Index = isa<ConstantInt>(ValIdx)
5620 ? cast<ConstantInt>(ValIdx)->getZExtValue()
5621 : -1;
5622 Type *PromotedType = getTransitionType();
5623
5624 StoreInst *ST = cast<StoreInst>(CombineInst);
5625 unsigned AS = ST->getPointerAddressSpace();
5626 unsigned Align = ST->getAlignment();
5627 // Check if this store is supported.
5628 if (!TLI.allowsMisalignedMemoryAccesses(
Mehdi Amini44ede332015-07-09 02:09:04 +00005629 TLI.getValueType(DL, ST->getValueOperand()->getType()), AS,
5630 Align)) {
Quentin Colombetc32615d2014-10-31 17:52:53 +00005631 // If this is not supported, there is no way we can combine
5632 // the extract with the store.
5633 return false;
5634 }
5635
5636 // The scalar chain of computation has to pay for the transition
5637 // scalar to vector.
5638 // The vector chain has to account for the combining cost.
5639 uint64_t ScalarCost =
5640 TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index);
5641 uint64_t VectorCost = StoreExtractCombineCost;
5642 for (const auto &Inst : InstsToBePromoted) {
5643 // Compute the cost.
5644 // By construction, all instructions being promoted are arithmetic ones.
5645 // Moreover, one argument is a constant that can be viewed as a splat
5646 // constant.
5647 Value *Arg0 = Inst->getOperand(0);
5648 bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) ||
5649 isa<ConstantFP>(Arg0);
5650 TargetTransformInfo::OperandValueKind Arg0OVK =
5651 IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue
5652 : TargetTransformInfo::OK_AnyValue;
5653 TargetTransformInfo::OperandValueKind Arg1OVK =
5654 !IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue
5655 : TargetTransformInfo::OK_AnyValue;
5656 ScalarCost += TTI.getArithmeticInstrCost(
5657 Inst->getOpcode(), Inst->getType(), Arg0OVK, Arg1OVK);
5658 VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType,
5659 Arg0OVK, Arg1OVK);
5660 }
5661 DEBUG(dbgs() << "Estimated cost of computation to be promoted:\nScalar: "
5662 << ScalarCost << "\nVector: " << VectorCost << '\n');
5663 return ScalarCost > VectorCost;
5664 }
5665
5666 /// \brief Generate a constant vector with \p Val with the same
5667 /// number of elements as the transition.
5668 /// \p UseSplat defines whether or not \p Val should be replicated
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00005669 /// across the whole vector.
Quentin Colombetc32615d2014-10-31 17:52:53 +00005670 /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>,
5671 /// otherwise we generate a vector with as many undef as possible:
5672 /// <undef, ..., undef, Val, undef, ..., undef> where \p Val is only
5673 /// used at the index of the extract.
5674 Value *getConstantVector(Constant *Val, bool UseSplat) const {
Eugene Zelenko900b6332017-08-29 22:32:07 +00005675 unsigned ExtractIdx = std::numeric_limits<unsigned>::max();
Quentin Colombetc32615d2014-10-31 17:52:53 +00005676 if (!UseSplat) {
5677 // If we cannot determine where the constant must be, we have to
5678 // use a splat constant.
5679 Value *ValExtractIdx = Transition->getOperand(getTransitionIdx());
5680 if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx))
5681 ExtractIdx = CstVal->getSExtValue();
5682 else
5683 UseSplat = true;
5684 }
5685
5686 unsigned End = getTransitionType()->getVectorNumElements();
5687 if (UseSplat)
5688 return ConstantVector::getSplat(End, Val);
5689
5690 SmallVector<Constant *, 4> ConstVec;
5691 UndefValue *UndefVal = UndefValue::get(Val->getType());
5692 for (unsigned Idx = 0; Idx != End; ++Idx) {
5693 if (Idx == ExtractIdx)
5694 ConstVec.push_back(Val);
5695 else
5696 ConstVec.push_back(UndefVal);
5697 }
5698 return ConstantVector::get(ConstVec);
5699 }
5700
5701 /// \brief Check if promoting to a vector type an operand at \p OperandIdx
5702 /// in \p Use can trigger undefined behavior.
5703 static bool canCauseUndefinedBehavior(const Instruction *Use,
5704 unsigned OperandIdx) {
5705 // This is not safe to introduce undef when the operand is on
5706 // the right hand side of a division-like instruction.
5707 if (OperandIdx != 1)
5708 return false;
5709 switch (Use->getOpcode()) {
5710 default:
5711 return false;
5712 case Instruction::SDiv:
5713 case Instruction::UDiv:
5714 case Instruction::SRem:
5715 case Instruction::URem:
5716 return true;
5717 case Instruction::FDiv:
5718 case Instruction::FRem:
5719 return !Use->hasNoNaNs();
5720 }
5721 llvm_unreachable(nullptr);
5722 }
5723
5724public:
Mehdi Amini44ede332015-07-09 02:09:04 +00005725 VectorPromoteHelper(const DataLayout &DL, const TargetLowering &TLI,
5726 const TargetTransformInfo &TTI, Instruction *Transition,
5727 unsigned CombineCost)
5728 : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition),
Eugene Zelenko900b6332017-08-29 22:32:07 +00005729 StoreExtractCombineCost(CombineCost) {
Quentin Colombetc32615d2014-10-31 17:52:53 +00005730 assert(Transition && "Do not know how to promote null");
5731 }
5732
5733 /// \brief Check if we can promote \p ToBePromoted to \p Type.
5734 bool canPromote(const Instruction *ToBePromoted) const {
5735 // We could support CastInst too.
5736 return isa<BinaryOperator>(ToBePromoted);
5737 }
5738
5739 /// \brief Check if it is profitable to promote \p ToBePromoted
5740 /// by moving downward the transition through.
5741 bool shouldPromote(const Instruction *ToBePromoted) const {
5742 // Promote only if all the operands can be statically expanded.
5743 // Indeed, we do not want to introduce any new kind of transitions.
5744 for (const Use &U : ToBePromoted->operands()) {
5745 const Value *Val = U.get();
5746 if (Val == getEndOfTransition()) {
5747 // If the use is a division and the transition is on the rhs,
5748 // we cannot promote the operation, otherwise we may create a
5749 // division by zero.
5750 if (canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()))
5751 return false;
5752 continue;
5753 }
5754 if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) &&
5755 !isa<ConstantFP>(Val))
5756 return false;
5757 }
5758 // Check that the resulting operation is legal.
5759 int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode());
5760 if (!ISDOpcode)
5761 return false;
5762 return StressStoreExtract ||
Ahmed Bougacha026600d2014-11-12 23:05:03 +00005763 TLI.isOperationLegalOrCustom(
Mehdi Amini44ede332015-07-09 02:09:04 +00005764 ISDOpcode, TLI.getValueType(DL, getTransitionType(), true));
Quentin Colombetc32615d2014-10-31 17:52:53 +00005765 }
5766
5767 /// \brief Check whether or not \p Use can be combined
5768 /// with the transition.
5769 /// I.e., is it possible to do Use(Transition) => AnotherUse?
5770 bool canCombine(const Instruction *Use) { return isa<StoreInst>(Use); }
5771
5772 /// \brief Record \p ToBePromoted as part of the chain to be promoted.
5773 void enqueueForPromotion(Instruction *ToBePromoted) {
5774 InstsToBePromoted.push_back(ToBePromoted);
5775 }
5776
5777 /// \brief Set the instruction that will be combined with the transition.
5778 void recordCombineInstruction(Instruction *ToBeCombined) {
5779 assert(canCombine(ToBeCombined) && "Unsupported instruction to combine");
5780 CombineInst = ToBeCombined;
5781 }
5782
5783 /// \brief Promote all the instructions enqueued for promotion if it is
5784 /// is profitable.
5785 /// \return True if the promotion happened, false otherwise.
5786 bool promote() {
5787 // Check if there is something to promote.
5788 // Right now, if we do not have anything to combine with,
5789 // we assume the promotion is not profitable.
5790 if (InstsToBePromoted.empty() || !CombineInst)
5791 return false;
5792
5793 // Check cost.
5794 if (!StressStoreExtract && !isProfitableToPromote())
5795 return false;
5796
5797 // Promote.
5798 for (auto &ToBePromoted : InstsToBePromoted)
5799 promoteImpl(ToBePromoted);
5800 InstsToBePromoted.clear();
5801 return true;
5802 }
5803};
Eugene Zelenko900b6332017-08-29 22:32:07 +00005804
5805} // end anonymous namespace
Quentin Colombetc32615d2014-10-31 17:52:53 +00005806
5807void VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) {
5808 // At this point, we know that all the operands of ToBePromoted but Def
5809 // can be statically promoted.
5810 // For Def, we need to use its parameter in ToBePromoted:
5811 // b = ToBePromoted ty1 a
5812 // Def = Transition ty1 b to ty2
5813 // Move the transition down.
5814 // 1. Replace all uses of the promoted operation by the transition.
5815 // = ... b => = ... Def.
5816 assert(ToBePromoted->getType() == Transition->getType() &&
5817 "The type of the result of the transition does not match "
5818 "the final type");
5819 ToBePromoted->replaceAllUsesWith(Transition);
5820 // 2. Update the type of the uses.
5821 // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def.
5822 Type *TransitionTy = getTransitionType();
5823 ToBePromoted->mutateType(TransitionTy);
5824 // 3. Update all the operands of the promoted operation with promoted
5825 // operands.
5826 // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a.
5827 for (Use &U : ToBePromoted->operands()) {
5828 Value *Val = U.get();
5829 Value *NewVal = nullptr;
5830 if (Val == Transition)
5831 NewVal = Transition->getOperand(getTransitionOriginalValueIdx());
5832 else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) ||
5833 isa<ConstantFP>(Val)) {
5834 // Use a splat constant if it is not safe to use undef.
5835 NewVal = getConstantVector(
5836 cast<Constant>(Val),
5837 isa<UndefValue>(Val) ||
5838 canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()));
5839 } else
Craig Topperd3c02f12015-01-05 10:15:49 +00005840 llvm_unreachable("Did you modified shouldPromote and forgot to update "
5841 "this?");
Quentin Colombetc32615d2014-10-31 17:52:53 +00005842 ToBePromoted->setOperand(U.getOperandNo(), NewVal);
5843 }
Sanjay Patel674d2c22017-08-29 14:07:48 +00005844 Transition->moveAfter(ToBePromoted);
Quentin Colombetc32615d2014-10-31 17:52:53 +00005845 Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted);
5846}
5847
5848/// Some targets can do store(extractelement) with one instruction.
5849/// Try to push the extractelement towards the stores when the target
5850/// has this feature and this is profitable.
Sanjay Patelfc580a62015-09-21 23:03:16 +00005851bool CodeGenPrepare::optimizeExtractElementInst(Instruction *Inst) {
Eugene Zelenko900b6332017-08-29 22:32:07 +00005852 unsigned CombineCost = std::numeric_limits<unsigned>::max();
Quentin Colombetc32615d2014-10-31 17:52:53 +00005853 if (DisableStoreExtract || !TLI ||
5854 (!StressStoreExtract &&
5855 !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(),
5856 Inst->getOperand(1), CombineCost)))
5857 return false;
5858
5859 // At this point we know that Inst is a vector to scalar transition.
5860 // Try to move it down the def-use chain, until:
5861 // - We can combine the transition with its single use
5862 // => we got rid of the transition.
5863 // - We escape the current basic block
5864 // => we would need to check that we are moving it at a cheaper place and
5865 // we do not do that for now.
5866 BasicBlock *Parent = Inst->getParent();
5867 DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n');
Mehdi Amini44ede332015-07-09 02:09:04 +00005868 VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost);
Quentin Colombetc32615d2014-10-31 17:52:53 +00005869 // If the transition has more than one use, assume this is not going to be
5870 // beneficial.
5871 while (Inst->hasOneUse()) {
5872 Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin());
5873 DEBUG(dbgs() << "Use: " << *ToBePromoted << '\n');
5874
5875 if (ToBePromoted->getParent() != Parent) {
5876 DEBUG(dbgs() << "Instruction to promote is in a different block ("
5877 << ToBePromoted->getParent()->getName()
5878 << ") than the transition (" << Parent->getName() << ").\n");
5879 return false;
5880 }
5881
5882 if (VPH.canCombine(ToBePromoted)) {
5883 DEBUG(dbgs() << "Assume " << *Inst << '\n'
5884 << "will be combined with: " << *ToBePromoted << '\n');
5885 VPH.recordCombineInstruction(ToBePromoted);
5886 bool Changed = VPH.promote();
5887 NumStoreExtractExposed += Changed;
5888 return Changed;
5889 }
5890
5891 DEBUG(dbgs() << "Try promoting.\n");
5892 if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted))
5893 return false;
5894
5895 DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n");
5896
5897 VPH.enqueueForPromotion(ToBePromoted);
5898 Inst = ToBePromoted;
5899 }
5900 return false;
5901}
5902
Wei Mia2f0b592016-12-22 19:44:45 +00005903/// For the instruction sequence of store below, F and I values
5904/// are bundled together as an i64 value before being stored into memory.
5905/// Sometimes it is more efficent to generate separate stores for F and I,
5906/// which can remove the bitwise instructions or sink them to colder places.
5907///
5908/// (store (or (zext (bitcast F to i32) to i64),
5909/// (shl (zext I to i64), 32)), addr) -->
5910/// (store F, addr) and (store I, addr+4)
5911///
5912/// Similarly, splitting for other merged store can also be beneficial, like:
5913/// For pair of {i32, i32}, i64 store --> two i32 stores.
5914/// For pair of {i32, i16}, i64 store --> two i32 stores.
5915/// For pair of {i16, i16}, i32 store --> two i16 stores.
5916/// For pair of {i16, i8}, i32 store --> two i16 stores.
5917/// For pair of {i8, i8}, i16 store --> two i8 stores.
5918///
5919/// We allow each target to determine specifically which kind of splitting is
5920/// supported.
5921///
5922/// The store patterns are commonly seen from the simple code snippet below
5923/// if only std::make_pair(...) is sroa transformed before inlined into hoo.
5924/// void goo(const std::pair<int, float> &);
5925/// hoo() {
5926/// ...
5927/// goo(std::make_pair(tmp, ftmp));
5928/// ...
5929/// }
5930///
5931/// Although we already have similar splitting in DAG Combine, we duplicate
5932/// it in CodeGenPrepare to catch the case in which pattern is across
5933/// multiple BBs. The logic in DAG Combine is kept to catch case generated
5934/// during code expansion.
5935static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL,
5936 const TargetLowering &TLI) {
5937 // Handle simple but common cases only.
5938 Type *StoreType = SI.getValueOperand()->getType();
5939 if (DL.getTypeStoreSizeInBits(StoreType) != DL.getTypeSizeInBits(StoreType) ||
5940 DL.getTypeSizeInBits(StoreType) == 0)
5941 return false;
5942
5943 unsigned HalfValBitSize = DL.getTypeSizeInBits(StoreType) / 2;
5944 Type *SplitStoreType = Type::getIntNTy(SI.getContext(), HalfValBitSize);
5945 if (DL.getTypeStoreSizeInBits(SplitStoreType) !=
5946 DL.getTypeSizeInBits(SplitStoreType))
5947 return false;
5948
5949 // Match the following patterns:
5950 // (store (or (zext LValue to i64),
5951 // (shl (zext HValue to i64), 32)), HalfValBitSize)
5952 // or
5953 // (store (or (shl (zext HValue to i64), 32)), HalfValBitSize)
5954 // (zext LValue to i64),
5955 // Expect both operands of OR and the first operand of SHL have only
5956 // one use.
5957 Value *LValue, *HValue;
5958 if (!match(SI.getValueOperand(),
5959 m_c_Or(m_OneUse(m_ZExt(m_Value(LValue))),
5960 m_OneUse(m_Shl(m_OneUse(m_ZExt(m_Value(HValue))),
5961 m_SpecificInt(HalfValBitSize))))))
5962 return false;
5963
5964 // Check LValue and HValue are int with size less or equal than 32.
5965 if (!LValue->getType()->isIntegerTy() ||
5966 DL.getTypeSizeInBits(LValue->getType()) > HalfValBitSize ||
5967 !HValue->getType()->isIntegerTy() ||
5968 DL.getTypeSizeInBits(HValue->getType()) > HalfValBitSize)
5969 return false;
5970
5971 // If LValue/HValue is a bitcast instruction, use the EVT before bitcast
5972 // as the input of target query.
5973 auto *LBC = dyn_cast<BitCastInst>(LValue);
5974 auto *HBC = dyn_cast<BitCastInst>(HValue);
5975 EVT LowTy = LBC ? EVT::getEVT(LBC->getOperand(0)->getType())
5976 : EVT::getEVT(LValue->getType());
5977 EVT HighTy = HBC ? EVT::getEVT(HBC->getOperand(0)->getType())
5978 : EVT::getEVT(HValue->getType());
5979 if (!ForceSplitStore && !TLI.isMultiStoresCheaperThanBitsMerge(LowTy, HighTy))
5980 return false;
5981
5982 // Start to split store.
5983 IRBuilder<> Builder(SI.getContext());
5984 Builder.SetInsertPoint(&SI);
5985
5986 // If LValue/HValue is a bitcast in another BB, create a new one in current
5987 // BB so it may be merged with the splitted stores by dag combiner.
5988 if (LBC && LBC->getParent() != SI.getParent())
5989 LValue = Builder.CreateBitCast(LBC->getOperand(0), LBC->getType());
5990 if (HBC && HBC->getParent() != SI.getParent())
5991 HValue = Builder.CreateBitCast(HBC->getOperand(0), HBC->getType());
5992
5993 auto CreateSplitStore = [&](Value *V, bool Upper) {
5994 V = Builder.CreateZExtOrBitCast(V, SplitStoreType);
5995 Value *Addr = Builder.CreateBitCast(
5996 SI.getOperand(1),
5997 SplitStoreType->getPointerTo(SI.getPointerAddressSpace()));
5998 if (Upper)
5999 Addr = Builder.CreateGEP(
6000 SplitStoreType, Addr,
6001 ConstantInt::get(Type::getInt32Ty(SI.getContext()), 1));
6002 Builder.CreateAlignedStore(
6003 V, Addr, Upper ? SI.getAlignment() / 2 : SI.getAlignment());
6004 };
6005
6006 CreateSplitStore(LValue, false);
6007 CreateSplitStore(HValue, true);
6008
6009 // Delete the old store.
6010 SI.eraseFromParent();
6011 return true;
6012}
6013
Hiroshi Yamauchi93644322017-09-11 17:52:08 +00006014// Return true if the GEP has two operands, the first operand is of a sequential
6015// type, and the second operand is a constant.
6016static bool GEPSequentialConstIndexed(GetElementPtrInst *GEP) {
6017 gep_type_iterator I = gep_type_begin(*GEP);
6018 return GEP->getNumOperands() == 2 &&
6019 I.isSequential() &&
6020 isa<ConstantInt>(GEP->getOperand(1));
6021}
6022
6023// Try unmerging GEPs to reduce liveness interference (register pressure) across
6024// IndirectBr edges. Since IndirectBr edges tend to touch on many blocks,
6025// reducing liveness interference across those edges benefits global register
6026// allocation. Currently handles only certain cases.
6027//
6028// For example, unmerge %GEPI and %UGEPI as below.
6029//
6030// ---------- BEFORE ----------
6031// SrcBlock:
6032// ...
6033// %GEPIOp = ...
6034// ...
6035// %GEPI = gep %GEPIOp, Idx
6036// ...
6037// indirectbr ... [ label %DstB0, label %DstB1, ... label %DstBi ... ]
6038// (* %GEPI is alive on the indirectbr edges due to other uses ahead)
6039// (* %GEPIOp is alive on the indirectbr edges only because of it's used by
6040// %UGEPI)
6041//
6042// DstB0: ... (there may be a gep similar to %UGEPI to be unmerged)
6043// DstB1: ... (there may be a gep similar to %UGEPI to be unmerged)
6044// ...
6045//
6046// DstBi:
6047// ...
6048// %UGEPI = gep %GEPIOp, UIdx
6049// ...
6050// ---------------------------
6051//
6052// ---------- AFTER ----------
6053// SrcBlock:
6054// ... (same as above)
6055// (* %GEPI is still alive on the indirectbr edges)
6056// (* %GEPIOp is no longer alive on the indirectbr edges as a result of the
6057// unmerging)
6058// ...
6059//
6060// DstBi:
6061// ...
6062// %UGEPI = gep %GEPI, (UIdx-Idx)
6063// ...
6064// ---------------------------
6065//
6066// The register pressure on the IndirectBr edges is reduced because %GEPIOp is
6067// no longer alive on them.
6068//
6069// We try to unmerge GEPs here in CodGenPrepare, as opposed to limiting merging
6070// of GEPs in the first place in InstCombiner::visitGetElementPtrInst() so as
6071// not to disable further simplications and optimizations as a result of GEP
6072// merging.
6073//
6074// Note this unmerging may increase the length of the data flow critical path
6075// (the path from %GEPIOp to %UGEPI would go through %GEPI), which is a tradeoff
6076// between the register pressure and the length of data-flow critical
6077// path. Restricting this to the uncommon IndirectBr case would minimize the
6078// impact of potentially longer critical path, if any, and the impact on compile
6079// time.
6080static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI,
6081 const TargetTransformInfo *TTI) {
6082 BasicBlock *SrcBlock = GEPI->getParent();
6083 // Check that SrcBlock ends with an IndirectBr. If not, give up. The common
6084 // (non-IndirectBr) cases exit early here.
6085 if (!isa<IndirectBrInst>(SrcBlock->getTerminator()))
6086 return false;
6087 // Check that GEPI is a simple gep with a single constant index.
6088 if (!GEPSequentialConstIndexed(GEPI))
6089 return false;
6090 ConstantInt *GEPIIdx = cast<ConstantInt>(GEPI->getOperand(1));
6091 // Check that GEPI is a cheap one.
6092 if (TTI->getIntImmCost(GEPIIdx->getValue(), GEPIIdx->getType())
6093 > TargetTransformInfo::TCC_Basic)
6094 return false;
6095 Value *GEPIOp = GEPI->getOperand(0);
6096 // Check that GEPIOp is an instruction that's also defined in SrcBlock.
6097 if (!isa<Instruction>(GEPIOp))
6098 return false;
6099 auto *GEPIOpI = cast<Instruction>(GEPIOp);
6100 if (GEPIOpI->getParent() != SrcBlock)
6101 return false;
6102 // Check that GEP is used outside the block, meaning it's alive on the
6103 // IndirectBr edge(s).
6104 if (find_if(GEPI->users(), [&](User *Usr) {
6105 if (auto *I = dyn_cast<Instruction>(Usr)) {
6106 if (I->getParent() != SrcBlock) {
6107 return true;
6108 }
6109 }
6110 return false;
6111 }) == GEPI->users().end())
6112 return false;
6113 // The second elements of the GEP chains to be unmerged.
6114 std::vector<GetElementPtrInst *> UGEPIs;
6115 // Check each user of GEPIOp to check if unmerging would make GEPIOp not alive
6116 // on IndirectBr edges.
6117 for (User *Usr : GEPIOp->users()) {
6118 if (Usr == GEPI) continue;
6119 // Check if Usr is an Instruction. If not, give up.
6120 if (!isa<Instruction>(Usr))
6121 return false;
6122 auto *UI = cast<Instruction>(Usr);
6123 // Check if Usr in the same block as GEPIOp, which is fine, skip.
6124 if (UI->getParent() == SrcBlock)
6125 continue;
6126 // Check if Usr is a GEP. If not, give up.
6127 if (!isa<GetElementPtrInst>(Usr))
6128 return false;
6129 auto *UGEPI = cast<GetElementPtrInst>(Usr);
6130 // Check if UGEPI is a simple gep with a single constant index and GEPIOp is
6131 // the pointer operand to it. If so, record it in the vector. If not, give
6132 // up.
6133 if (!GEPSequentialConstIndexed(UGEPI))
6134 return false;
6135 if (UGEPI->getOperand(0) != GEPIOp)
6136 return false;
6137 if (GEPIIdx->getType() !=
6138 cast<ConstantInt>(UGEPI->getOperand(1))->getType())
6139 return false;
6140 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));
6141 if (TTI->getIntImmCost(UGEPIIdx->getValue(), UGEPIIdx->getType())
6142 > TargetTransformInfo::TCC_Basic)
6143 return false;
6144 UGEPIs.push_back(UGEPI);
6145 }
6146 if (UGEPIs.size() == 0)
6147 return false;
6148 // Check the materializing cost of (Uidx-Idx).
6149 for (GetElementPtrInst *UGEPI : UGEPIs) {
6150 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));
6151 APInt NewIdx = UGEPIIdx->getValue() - GEPIIdx->getValue();
6152 unsigned ImmCost = TTI->getIntImmCost(NewIdx, GEPIIdx->getType());
6153 if (ImmCost > TargetTransformInfo::TCC_Basic)
6154 return false;
6155 }
6156 // Now unmerge between GEPI and UGEPIs.
6157 for (GetElementPtrInst *UGEPI : UGEPIs) {
6158 UGEPI->setOperand(0, GEPI);
6159 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));
6160 Constant *NewUGEPIIdx =
6161 ConstantInt::get(GEPIIdx->getType(),
6162 UGEPIIdx->getValue() - GEPIIdx->getValue());
6163 UGEPI->setOperand(1, NewUGEPIIdx);
6164 // If GEPI is not inbounds but UGEPI is inbounds, change UGEPI to not
6165 // inbounds to avoid UB.
6166 if (!GEPI->isInBounds()) {
6167 UGEPI->setIsInBounds(false);
6168 }
6169 }
6170 // After unmerging, verify that GEPIOp is actually only used in SrcBlock (not
6171 // alive on IndirectBr edges).
6172 assert(find_if(GEPIOp->users(), [&](User *Usr) {
6173 return cast<Instruction>(Usr)->getParent() != SrcBlock;
6174 }) == GEPIOp->users().end() && "GEPIOp is used outside SrcBlock");
6175 return true;
6176}
6177
Sanjay Patel3b8974b2017-06-08 20:00:09 +00006178bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) {
Ahmed Bougachaf3299142015-06-17 20:44:32 +00006179 // Bail out if we inserted the instruction to prevent optimizations from
6180 // stepping on each other's toes.
6181 if (InsertedInsts.count(I))
6182 return false;
6183
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006184 if (PHINode *P = dyn_cast<PHINode>(I)) {
6185 // It is possible for very late stage optimizations (such as SimplifyCFG)
6186 // to introduce PHI nodes too late to be cleaned up. If we detect such a
6187 // trivial PHI, go ahead and zap it here.
Daniel Berlin4d0fe642017-04-28 19:55:38 +00006188 if (Value *V = SimplifyInstruction(P, {*DL, TLInfo})) {
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006189 P->replaceAllUsesWith(V);
6190 P->eraseFromParent();
6191 ++NumPHIsElim;
Chris Lattneree588de2011-01-15 07:29:01 +00006192 return true;
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006193 }
Chris Lattneree588de2011-01-15 07:29:01 +00006194 return false;
6195 }
Nadav Rotem465834c2012-07-24 10:51:42 +00006196
Chris Lattneree588de2011-01-15 07:29:01 +00006197 if (CastInst *CI = dyn_cast<CastInst>(I)) {
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006198 // If the source of the cast is a constant, then this should have
6199 // already been constant folded. The only reason NOT to constant fold
6200 // it is if something (e.g. LSR) was careful to place the constant
6201 // evaluation in a block other than then one that uses it (e.g. to hoist
6202 // the address of globals out of a loop). If this is the case, we don't
6203 // want to forward-subst the cast.
6204 if (isa<Constant>(CI->getOperand(0)))
6205 return false;
6206
Mehdi Amini44ede332015-07-09 02:09:04 +00006207 if (TLI && OptimizeNoopCopyExpression(CI, *TLI, *DL))
Chris Lattneree588de2011-01-15 07:29:01 +00006208 return true;
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006209
Chris Lattneree588de2011-01-15 07:29:01 +00006210 if (isa<ZExtInst>(I) || isa<SExtInst>(I)) {
Manuel Jacoba7c48f92014-03-13 13:36:25 +00006211 /// Sink a zext or sext into its user blocks if the target type doesn't
6212 /// fit in one register
Mehdi Amini44ede332015-07-09 02:09:04 +00006213 if (TLI &&
6214 TLI->getTypeAction(CI->getContext(),
6215 TLI->getValueType(*DL, CI->getType())) ==
6216 TargetLowering::TypeExpandInteger) {
Manuel Jacoba7c48f92014-03-13 13:36:25 +00006217 return SinkCast(CI);
6218 } else {
Jun Bum Limdee55652017-04-03 19:20:07 +00006219 bool MadeChange = optimizeExt(I);
Sanjay Patelfc580a62015-09-21 23:03:16 +00006220 return MadeChange | optimizeExtUses(I);
Manuel Jacoba7c48f92014-03-13 13:36:25 +00006221 }
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006222 }
Chris Lattneree588de2011-01-15 07:29:01 +00006223 return false;
6224 }
Nadav Rotem465834c2012-07-24 10:51:42 +00006225
Chris Lattneree588de2011-01-15 07:29:01 +00006226 if (CmpInst *CI = dyn_cast<CmpInst>(I))
Hal Finkeldecb0242014-01-02 21:13:43 +00006227 if (!TLI || !TLI->hasMultipleConditionRegisters())
Peter Zotovf87e5502016-04-03 17:11:53 +00006228 return OptimizeCmpExpression(CI, TLI);
Nadav Rotem465834c2012-07-24 10:51:42 +00006229
Chris Lattneree588de2011-01-15 07:29:01 +00006230 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
Sanjoy Das00757272016-12-16 20:29:39 +00006231 LI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00006232 if (TLI) {
Geoff Berry5256fca2015-11-20 22:34:39 +00006233 bool Modified = optimizeLoadExt(LI);
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00006234 unsigned AS = LI->getPointerAddressSpace();
Geoff Berry5256fca2015-11-20 22:34:39 +00006235 Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS);
6236 return Modified;
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00006237 }
Hans Wennborgf3254832012-10-30 11:23:25 +00006238 return false;
Chris Lattneree588de2011-01-15 07:29:01 +00006239 }
Nadav Rotem465834c2012-07-24 10:51:42 +00006240
Chris Lattneree588de2011-01-15 07:29:01 +00006241 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
Wei Mia2f0b592016-12-22 19:44:45 +00006242 if (TLI && splitMergedValStore(*SI, *DL, *TLI))
6243 return true;
Sanjoy Das00757272016-12-16 20:29:39 +00006244 SI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00006245 if (TLI) {
6246 unsigned AS = SI->getPointerAddressSpace();
Sanjay Patelfc580a62015-09-21 23:03:16 +00006247 return optimizeMemoryInst(I, SI->getOperand(1),
Matt Arsenaultf72b49b2015-06-04 16:17:38 +00006248 SI->getOperand(0)->getType(), AS);
6249 }
Chris Lattneree588de2011-01-15 07:29:01 +00006250 return false;
6251 }
Nadav Rotem465834c2012-07-24 10:51:42 +00006252
Matt Arsenault02d915b2017-03-15 22:35:20 +00006253 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
6254 unsigned AS = RMW->getPointerAddressSpace();
6255 return optimizeMemoryInst(I, RMW->getPointerOperand(),
6256 RMW->getType(), AS);
6257 }
6258
6259 if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(I)) {
6260 unsigned AS = CmpX->getPointerAddressSpace();
6261 return optimizeMemoryInst(I, CmpX->getPointerOperand(),
6262 CmpX->getCompareOperand()->getType(), AS);
6263 }
6264
Yi Jiangd069f632014-04-21 19:34:27 +00006265 BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I);
6266
Geoff Berry5d534b62017-02-21 18:53:14 +00006267 if (BinOp && (BinOp->getOpcode() == Instruction::And) &&
6268 EnableAndCmpSinking && TLI)
6269 return sinkAndCmp0Expression(BinOp, *TLI, InsertedInsts);
6270
Yi Jiangd069f632014-04-21 19:34:27 +00006271 if (BinOp && (BinOp->getOpcode() == Instruction::AShr ||
6272 BinOp->getOpcode() == Instruction::LShr)) {
6273 ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1));
6274 if (TLI && CI && TLI->hasExtractBitsInsn())
Mehdi Amini44ede332015-07-09 02:09:04 +00006275 return OptimizeExtractBits(BinOp, CI, *TLI, *DL);
Yi Jiangd069f632014-04-21 19:34:27 +00006276
6277 return false;
6278 }
6279
Chris Lattneree588de2011-01-15 07:29:01 +00006280 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {
Cameron Zwarichd28c78e2011-01-06 02:44:52 +00006281 if (GEPI->hasAllZeroIndices()) {
6282 /// The GEP operand must be a pointer, so must its result -> BitCast
6283 Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(),
6284 GEPI->getName(), GEPI);
6285 GEPI->replaceAllUsesWith(NC);
6286 GEPI->eraseFromParent();
6287 ++NumGEPsElim;
Sanjay Patelfc580a62015-09-21 23:03:16 +00006288 optimizeInst(NC, ModifiedDT);
Chris Lattneree588de2011-01-15 07:29:01 +00006289 return true;
Cameron Zwarichd28c78e2011-01-06 02:44:52 +00006290 }
Hiroshi Yamauchi93644322017-09-11 17:52:08 +00006291 if (tryUnmergingGEPsAcrossIndirectBr(GEPI, TTI)) {
6292 return true;
6293 }
Chris Lattneree588de2011-01-15 07:29:01 +00006294 return false;
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006295 }
Nadav Rotem465834c2012-07-24 10:51:42 +00006296
Chris Lattneree588de2011-01-15 07:29:01 +00006297 if (CallInst *CI = dyn_cast<CallInst>(I))
Sanjay Patelfc580a62015-09-21 23:03:16 +00006298 return optimizeCallInst(CI, ModifiedDT);
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006299
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00006300 if (SelectInst *SI = dyn_cast<SelectInst>(I))
Sanjay Patelfc580a62015-09-21 23:03:16 +00006301 return optimizeSelectInst(SI);
Benjamin Kramer047d7ca2012-05-05 12:49:22 +00006302
Tim Northoveraeb8e062014-02-19 10:02:43 +00006303 if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
Sanjay Patelfc580a62015-09-21 23:03:16 +00006304 return optimizeShuffleVectorInst(SVI);
Tim Northoveraeb8e062014-02-19 10:02:43 +00006305
Sanjay Patel0ed9aea2015-11-02 23:22:49 +00006306 if (auto *Switch = dyn_cast<SwitchInst>(I))
6307 return optimizeSwitchInst(Switch);
6308
Quentin Colombetc32615d2014-10-31 17:52:53 +00006309 if (isa<ExtractElementInst>(I))
Sanjay Patelfc580a62015-09-21 23:03:16 +00006310 return optimizeExtractElementInst(I);
Quentin Colombetc32615d2014-10-31 17:52:53 +00006311
Chris Lattneree588de2011-01-15 07:29:01 +00006312 return false;
Cameron Zwarich14ac8652011-01-06 02:37:26 +00006313}
6314
James Molloyf01488e2016-01-15 09:20:19 +00006315/// Given an OR instruction, check to see if this is a bitreverse
6316/// idiom. If so, insert the new intrinsic and return true.
6317static bool makeBitReverse(Instruction &I, const DataLayout &DL,
6318 const TargetLowering &TLI) {
6319 if (!I.getType()->isIntegerTy() ||
6320 !TLI.isOperationLegalOrCustom(ISD::BITREVERSE,
6321 TLI.getValueType(DL, I.getType(), true)))
6322 return false;
6323
6324 SmallVector<Instruction*, 4> Insts;
Chad Rosiera00df492016-05-25 16:22:14 +00006325 if (!recognizeBSwapOrBitReverseIdiom(&I, false, true, Insts))
James Molloyf01488e2016-01-15 09:20:19 +00006326 return false;
6327 Instruction *LastInst = Insts.back();
6328 I.replaceAllUsesWith(LastInst);
6329 RecursivelyDeleteTriviallyDeadInstructions(&I);
6330 return true;
6331}
6332
Chris Lattnerf2836d12007-03-31 04:06:36 +00006333// In this pass we look for GEP and cast instructions that are used
6334// across basic blocks and rewrite them to improve basic-block-at-a-time
6335// selection.
Sanjay Patel3b8974b2017-06-08 20:00:09 +00006336bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, bool &ModifiedDT) {
Cameron Zwarichce3b9302011-01-06 00:42:50 +00006337 SunkAddrs.clear();
Cameron Zwarich5dd2aa22011-03-02 03:31:46 +00006338 bool MadeChange = false;
Eric Christopherc1ea1492008-09-24 05:32:41 +00006339
Chris Lattner7a277142011-01-15 07:14:54 +00006340 CurInstIterator = BB.begin();
Elena Demikhovsky87700a72014-12-28 08:54:45 +00006341 while (CurInstIterator != BB.end()) {
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00006342 MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT);
Elena Demikhovsky87700a72014-12-28 08:54:45 +00006343 if (ModifiedDT)
6344 return true;
6345 }
Benjamin Kramer455fa352012-11-23 19:17:06 +00006346
James Molloyf01488e2016-01-15 09:20:19 +00006347 bool MadeBitReverse = true;
6348 while (TLI && MadeBitReverse) {
6349 MadeBitReverse = false;
6350 for (auto &I : reverse(BB)) {
6351 if (makeBitReverse(I, *DL, *TLI)) {
6352 MadeBitReverse = MadeChange = true;
George Burgess IVd4febd12016-03-22 21:25:08 +00006353 ModifiedDT = true;
James Molloyf01488e2016-01-15 09:20:19 +00006354 break;
6355 }
6356 }
6357 }
James Molloy3ef84c42016-01-15 10:36:01 +00006358 MadeChange |= dupRetToEnableTailCallOpts(&BB);
Junmo Park7d6c5f12016-01-28 09:42:39 +00006359
Chris Lattnerf2836d12007-03-31 04:06:36 +00006360 return MadeChange;
6361}
Devang Patel53771ba2011-08-18 00:50:51 +00006362
6363// llvm.dbg.value is far away from the value then iSel may not be able
Nadav Rotem465834c2012-07-24 10:51:42 +00006364// handle it properly. iSel will drop llvm.dbg.value if it can not
Devang Patel53771ba2011-08-18 00:50:51 +00006365// find a node corresponding to the value.
Sanjay Patelfc580a62015-09-21 23:03:16 +00006366bool CodeGenPrepare::placeDbgValues(Function &F) {
Devang Patel53771ba2011-08-18 00:50:51 +00006367 bool MadeChange = false;
Duncan P. N. Exon Smith5914a972015-01-08 20:44:33 +00006368 for (BasicBlock &BB : F) {
Craig Topperc0196b12014-04-14 00:51:57 +00006369 Instruction *PrevNonDbgInst = nullptr;
Duncan P. N. Exon Smith5914a972015-01-08 20:44:33 +00006370 for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) {
Duncan P. N. Exon Smithd83547a2015-10-09 18:44:40 +00006371 Instruction *Insn = &*BI++;
Devang Patel53771ba2011-08-18 00:50:51 +00006372 DbgValueInst *DVI = dyn_cast<DbgValueInst>(Insn);
Adrian Prantl32da8892014-04-25 20:49:25 +00006373 // Leave dbg.values that refer to an alloca alone. These
Craig Topper87e715f2017-11-07 20:56:17 +00006374 // intrinsics describe the address of a variable (= the alloca)
Adrian Prantl32da8892014-04-25 20:49:25 +00006375 // being taken. They should not be moved next to the alloca
6376 // (and to the beginning of the scope), but rather stay close to
6377 // where said address is used.
6378 if (!DVI || (DVI->getValue() && isa<AllocaInst>(DVI->getValue()))) {
Devang Patel53771ba2011-08-18 00:50:51 +00006379 PrevNonDbgInst = Insn;
6380 continue;
6381 }
6382
6383 Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue());
6384 if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) {
Reid Kleckner8de1fe22015-12-08 23:00:03 +00006385 // If VI is a phi in a block with an EHPad terminator, we can't insert
6386 // after it.
6387 if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad())
6388 continue;
Devang Patel53771ba2011-08-18 00:50:51 +00006389 DEBUG(dbgs() << "Moving Debug Value before :\n" << *DVI << ' ' << *VI);
6390 DVI->removeFromParent();
Reid Klecknere18f92b2015-12-08 22:33:23 +00006391 if (isa<PHINode>(VI))
6392 DVI->insertBefore(&*VI->getParent()->getFirstInsertionPt());
6393 else
6394 DVI->insertAfter(VI);
Devang Patel53771ba2011-08-18 00:50:51 +00006395 MadeChange = true;
6396 ++NumDbgValueMoved;
6397 }
6398 }
6399 }
6400 return MadeChange;
6401}
Tim Northovercea0abb2014-03-29 08:22:29 +00006402
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006403/// \brief Scale down both weights to fit into uint32_t.
6404static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
6405 uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
Eugene Zelenko900b6332017-08-29 22:32:07 +00006406 uint32_t Scale = (NewMax / std::numeric_limits<uint32_t>::max()) + 1;
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006407 NewTrue = NewTrue / Scale;
6408 NewFalse = NewFalse / Scale;
6409}
6410
6411/// \brief Some targets prefer to split a conditional branch like:
6412/// \code
6413/// %0 = icmp ne i32 %a, 0
6414/// %1 = icmp ne i32 %b, 0
6415/// %or.cond = or i1 %0, %1
6416/// br i1 %or.cond, label %TrueBB, label %FalseBB
6417/// \endcode
6418/// into multiple branch instructions like:
6419/// \code
6420/// bb1:
6421/// %0 = icmp ne i32 %a, 0
6422/// br i1 %0, label %TrueBB, label %bb2
6423/// bb2:
6424/// %1 = icmp ne i32 %b, 0
6425/// br i1 %1, label %TrueBB, label %FalseBB
6426/// \endcode
6427/// This usually allows instruction selection to do even further optimizations
6428/// and combine the compare with the branch instruction. Currently this is
6429/// applied for targets which have "cheap" jump instructions.
6430///
6431/// FIXME: Remove the (equivalent?) implementation in SelectionDAG.
6432///
6433bool CodeGenPrepare::splitBranchCondition(Function &F) {
David Blaikiedc3f01e2015-03-09 01:57:13 +00006434 if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive())
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006435 return false;
6436
6437 bool MadeChange = false;
6438 for (auto &BB : F) {
6439 // Does this BB end with the following?
6440 // %cond1 = icmp|fcmp|binary instruction ...
6441 // %cond2 = icmp|fcmp|binary instruction ...
6442 // %cond.or = or|and i1 %cond1, cond2
6443 // br i1 %cond.or label %dest1, label %dest2"
6444 BinaryOperator *LogicOp;
6445 BasicBlock *TBB, *FBB;
6446 if (!match(BB.getTerminator(), m_Br(m_OneUse(m_BinOp(LogicOp)), TBB, FBB)))
6447 continue;
6448
Sanjay Patel42574202015-09-02 19:23:23 +00006449 auto *Br1 = cast<BranchInst>(BB.getTerminator());
6450 if (Br1->getMetadata(LLVMContext::MD_unpredictable))
6451 continue;
6452
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006453 unsigned Opc;
Juergen Ributzka8bda7382014-12-09 17:50:10 +00006454 Value *Cond1, *Cond2;
6455 if (match(LogicOp, m_And(m_OneUse(m_Value(Cond1)),
6456 m_OneUse(m_Value(Cond2)))))
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006457 Opc = Instruction::And;
Juergen Ributzka8bda7382014-12-09 17:50:10 +00006458 else if (match(LogicOp, m_Or(m_OneUse(m_Value(Cond1)),
6459 m_OneUse(m_Value(Cond2)))))
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006460 Opc = Instruction::Or;
6461 else
6462 continue;
6463
6464 if (!match(Cond1, m_CombineOr(m_Cmp(), m_BinOp())) ||
6465 !match(Cond2, m_CombineOr(m_Cmp(), m_BinOp())) )
6466 continue;
6467
6468 DEBUG(dbgs() << "Before branch condition splitting\n"; BB.dump());
6469
6470 // Create a new BB.
Duncan P. N. Exon Smitha848c472016-02-21 19:52:15 +00006471 auto TmpBB =
6472 BasicBlock::Create(BB.getContext(), BB.getName() + ".cond.split",
6473 BB.getParent(), BB.getNextNode());
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006474
6475 // Update original basic block by using the first condition directly by the
6476 // branch instruction and removing the no longer needed and/or instruction.
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006477 Br1->setCondition(Cond1);
6478 LogicOp->eraseFromParent();
Juergen Ributzka8bda7382014-12-09 17:50:10 +00006479
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006480 // Depending on the conditon we have to either replace the true or the false
6481 // successor of the original branch instruction.
6482 if (Opc == Instruction::And)
6483 Br1->setSuccessor(0, TmpBB);
6484 else
6485 Br1->setSuccessor(1, TmpBB);
6486
6487 // Fill in the new basic block.
6488 auto *Br2 = IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB);
Juergen Ributzka8bda7382014-12-09 17:50:10 +00006489 if (auto *I = dyn_cast<Instruction>(Cond2)) {
6490 I->removeFromParent();
6491 I->insertBefore(Br2);
6492 }
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006493
6494 // Update PHI nodes in both successors. The original BB needs to be
Hiroshi Inoue6a391bb2017-06-27 10:35:37 +00006495 // replaced in one successor's PHI nodes, because the branch comes now from
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006496 // the newly generated BB (NewBB). In the other successor we need to add one
6497 // incoming edge to the PHI nodes, because both branch instructions target
6498 // now the same successor. Depending on the original branch condition
6499 // (and/or) we have to swap the successors (TrueDest, FalseDest), so that
Simon Pilgrimf2fbf432016-11-20 13:47:59 +00006500 // we perform the correct update for the PHI nodes.
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006501 // This doesn't change the successor order of the just created branch
6502 // instruction (or any other instruction).
6503 if (Opc == Instruction::Or)
6504 std::swap(TBB, FBB);
6505
6506 // Replace the old BB with the new BB.
6507 for (auto &I : *TBB) {
6508 PHINode *PN = dyn_cast<PHINode>(&I);
6509 if (!PN)
6510 break;
6511 int i;
6512 while ((i = PN->getBasicBlockIndex(&BB)) >= 0)
6513 PN->setIncomingBlock(i, TmpBB);
6514 }
6515
6516 // Add another incoming edge form the new BB.
6517 for (auto &I : *FBB) {
6518 PHINode *PN = dyn_cast<PHINode>(&I);
6519 if (!PN)
6520 break;
6521 auto *Val = PN->getIncomingValueForBlock(&BB);
6522 PN->addIncoming(Val, TmpBB);
6523 }
6524
6525 // Update the branch weights (from SelectionDAGBuilder::
6526 // FindMergedConditions).
6527 if (Opc == Instruction::Or) {
6528 // Codegen X | Y as:
6529 // BB1:
6530 // jmp_if_X TBB
6531 // jmp TmpBB
6532 // TmpBB:
6533 // jmp_if_Y TBB
6534 // jmp FBB
6535 //
6536
6537 // We have flexibility in setting Prob for BB1 and Prob for NewBB.
6538 // The requirement is that
6539 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
6540 // = TrueProb for orignal BB.
6541 // Assuming the orignal weights are A and B, one choice is to set BB1's
6542 // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
6543 // assumes that
6544 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
6545 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
6546 // TmpBB, but the math is more complicated.
6547 uint64_t TrueWeight, FalseWeight;
Sanjay Pateldc88bd62016-04-23 20:01:22 +00006548 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006549 uint64_t NewTrueWeight = TrueWeight;
6550 uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight;
6551 scaleWeights(NewTrueWeight, NewFalseWeight);
6552 Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext())
6553 .createBranchWeights(TrueWeight, FalseWeight));
6554
6555 NewTrueWeight = TrueWeight;
6556 NewFalseWeight = 2 * FalseWeight;
6557 scaleWeights(NewTrueWeight, NewFalseWeight);
6558 Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext())
6559 .createBranchWeights(TrueWeight, FalseWeight));
6560 }
6561 } else {
6562 // Codegen X & Y as:
6563 // BB1:
6564 // jmp_if_X TmpBB
6565 // jmp FBB
6566 // TmpBB:
6567 // jmp_if_Y TBB
6568 // jmp FBB
6569 //
6570 // This requires creation of TmpBB after CurBB.
6571
6572 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
6573 // The requirement is that
6574 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
6575 // = FalseProb for orignal BB.
6576 // Assuming the orignal weights are A and B, one choice is to set BB1's
6577 // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
6578 // assumes that
6579 // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
6580 uint64_t TrueWeight, FalseWeight;
Sanjay Pateldc88bd62016-04-23 20:01:22 +00006581 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006582 uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight;
6583 uint64_t NewFalseWeight = FalseWeight;
6584 scaleWeights(NewTrueWeight, NewFalseWeight);
6585 Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext())
6586 .createBranchWeights(TrueWeight, FalseWeight));
6587
6588 NewTrueWeight = 2 * TrueWeight;
6589 NewFalseWeight = FalseWeight;
6590 scaleWeights(NewTrueWeight, NewFalseWeight);
6591 Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext())
6592 .createBranchWeights(TrueWeight, FalseWeight));
6593 }
6594 }
6595
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006596 // Note: No point in getting fancy here, since the DT info is never
Quentin Colombet7bdd50d2015-03-18 23:17:28 +00006597 // available to CodeGenPrepare.
Juergen Ributzkac1bbcbb2014-12-09 16:36:13 +00006598 ModifiedDT = true;
6599
6600 MadeChange = true;
6601
6602 DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump();
6603 TmpBB->dump());
6604 }
6605 return MadeChange;
6606}