blob: bd1f21c69eba7be4219c4d21729f8ef70f1c187d [file] [log] [blame]
Dan Gohman0a40ad92009-04-16 03:18:22 +00001//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
Misha Brukmanb1c93172005-04-21 23:48:37 +00002//
Nate Begemanb18121e2004-10-18 21:08:22 +00003// The LLVM Compiler Infrastructure
4//
Chris Lattnerf3ebc3f2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Misha Brukmanb1c93172005-04-21 23:48:37 +00007//
Nate Begemanb18121e2004-10-18 21:08:22 +00008//===----------------------------------------------------------------------===//
9//
Dan Gohman97f70ad2009-05-19 20:37:36 +000010// This transformation analyzes and transforms the induction variables (and
11// computations derived from them) into forms suitable for efficient execution
12// on the target.
13//
Nate Begemanb18121e2004-10-18 21:08:22 +000014// This pass performs a strength reduction on array references inside loops that
Dan Gohman97f70ad2009-05-19 20:37:36 +000015// have as one or more of their components the loop induction variable, it
16// rewrites expressions to take advantage of scaled-index addressing modes
17// available on the target, and it performs a variety of other optimizations
18// related to loop induction variables.
Nate Begemanb18121e2004-10-18 21:08:22 +000019//
Dan Gohman45774ce2010-02-12 10:34:29 +000020// Terminology note: this code has a lot of handling for "post-increment" or
21// "post-inc" users. This is not talking about post-increment addressing modes;
22// it is instead talking about code like this:
23//
24// %i = phi [ 0, %entry ], [ %i.next, %latch ]
25// ...
26// %i.next = add %i, 1
27// %c = icmp eq %i.next, %n
28//
29// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
30// it's useful to think about these as the same register, with some uses using
Sanjoy Das7041fb12015-03-27 06:01:56 +000031// the value of the register before the add and some using it after. In this
Dan Gohman45774ce2010-02-12 10:34:29 +000032// example, the icmp is a post-increment user, since it uses %i.next, which is
33// the value of the induction variable after the increment. The other common
34// case of post-increment users is users outside the loop.
35//
36// TODO: More sophistication in the way Formulae are generated and filtered.
37//
38// TODO: Handle multiple loops at a time.
39//
Chandler Carruth26c59fa2013-01-07 14:41:08 +000040// TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead
41// of a GlobalValue?
Dan Gohman45774ce2010-02-12 10:34:29 +000042//
43// TODO: When truncation is free, truncate ICmp users' operands to make it a
44// smaller encoding (on x86 at least).
45//
46// TODO: When a negated register is used by an add (such as in a list of
47// multiple base registers, or as the increment expression in an addrec),
48// we may not actually need both reg and (-1 * reg) in registers; the
49// negation can be implemented by using a sub instead of an add. The
50// lack of support for taking this into consideration when making
51// register pressure decisions is partly worked around by the "Special"
52// use kind.
53//
Nate Begemanb18121e2004-10-18 21:08:22 +000054//===----------------------------------------------------------------------===//
55
Dehao Chen6132ee82016-07-18 21:41:50 +000056#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000057#include "llvm/ADT/APInt.h"
58#include "llvm/ADT/DenseMap.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000059#include "llvm/ADT/DenseSet.h"
Benjamin Kramer62fb0cf2014-03-15 17:17:48 +000060#include "llvm/ADT/Hashing.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000061#include "llvm/ADT/PointerIntPair.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +000062#include "llvm/ADT/STLExtras.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000063#include "llvm/ADT/SetVector.h"
64#include "llvm/ADT/SmallBitVector.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000065#include "llvm/ADT/SmallPtrSet.h"
66#include "llvm/ADT/SmallSet.h"
67#include "llvm/ADT/SmallVector.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000068#include "llvm/Analysis/IVUsers.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000069#include "llvm/Analysis/LoopInfo.h"
Devang Patelb0743b52007-03-06 21:14:09 +000070#include "llvm/Analysis/LoopPass.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000071#include "llvm/Analysis/ScalarEvolution.h"
Nate Begemane68bcd12005-07-30 00:15:07 +000072#include "llvm/Analysis/ScalarEvolutionExpander.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000073#include "llvm/Analysis/ScalarEvolutionExpressions.h"
74#include "llvm/Analysis/ScalarEvolutionNormalization.h"
Chandler Carruth26c59fa2013-01-07 14:41:08 +000075#include "llvm/Analysis/TargetTransformInfo.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000076#include "llvm/IR/BasicBlock.h"
77#include "llvm/IR/Constant.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000078#include "llvm/IR/Constants.h"
79#include "llvm/IR/DerivedTypes.h"
Chandler Carruth5ad5f152014-01-13 09:26:24 +000080#include "llvm/IR/Dominators.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000081#include "llvm/IR/GlobalValue.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +000082#include "llvm/IR/IRBuilder.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000083#include "llvm/IR/Instruction.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000084#include "llvm/IR/Instructions.h"
85#include "llvm/IR/IntrinsicInst.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +000086#include "llvm/IR/Module.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000087#include "llvm/IR/OperandTraits.h"
88#include "llvm/IR/Operator.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000089#include "llvm/IR/Type.h"
90#include "llvm/IR/Value.h"
Chandler Carruth4220e9c2014-03-04 11:17:44 +000091#include "llvm/IR/ValueHandle.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000092#include "llvm/Pass.h"
93#include "llvm/Support/Casting.h"
Andrew Trick58124392011-09-27 00:44:14 +000094#include "llvm/Support/CommandLine.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000095#include "llvm/Support/Compiler.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000096#include "llvm/Support/Debug.h"
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +000097#include "llvm/Support/ErrorHandling.h"
98#include "llvm/Support/MathExtras.h"
Daniel Dunbar6115b392009-07-26 09:48:23 +000099#include "llvm/Support/raw_ostream.h"
Dehao Chen6132ee82016-07-18 21:41:50 +0000100#include "llvm/Transforms/Scalar.h"
Chandler Carruth3bab7e12017-01-11 09:43:56 +0000101#include "llvm/Transforms/Scalar/LoopPassManager.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000102#include "llvm/Transforms/Utils/BasicBlockUtils.h"
103#include "llvm/Transforms/Utils/Local.h"
Jeff Cohenc5009912005-07-30 18:22:27 +0000104#include <algorithm>
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +0000105#include <cassert>
106#include <cstddef>
107#include <cstdint>
108#include <cstdlib>
109#include <iterator>
110#include <map>
111#include <tuple>
112#include <utility>
113
Nate Begemanb18121e2004-10-18 21:08:22 +0000114using namespace llvm;
115
Chandler Carruth964daaa2014-04-22 02:55:47 +0000116#define DEBUG_TYPE "loop-reduce"
117
Andrew Trick19f80c12012-04-18 04:00:10 +0000118/// MaxIVUsers is an arbitrary threshold that provides an early opportunitiy for
119/// bail out. This threshold is far beyond the number of users that LSR can
120/// conceivably solve, so it should not affect generated code, but catches the
121/// worst cases before LSR burns too much compile time and stack space.
122static const unsigned MaxIVUsers = 200;
123
Andrew Trickecbe22b2011-10-11 02:30:45 +0000124// Temporary flag to cleanup congruent phis after LSR phi expansion.
125// It's currently disabled until we can determine whether it's truly useful or
126// not. The flag should be removed after the v3.0 release.
Andrew Trick06f6c052012-01-07 07:08:17 +0000127// This is now needed for ivchains.
Benjamin Kramer7ba71be2011-11-26 23:01:57 +0000128static cl::opt<bool> EnablePhiElim(
Andrew Trick06f6c052012-01-07 07:08:17 +0000129 "enable-lsr-phielim", cl::Hidden, cl::init(true),
130 cl::desc("Enable LSR phi elimination"));
Andrew Trick58124392011-09-27 00:44:14 +0000131
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +0000132// The flag adds instruction count to solutions cost comparision.
133static cl::opt<bool> InsnsCost(
134 "lsr-insns-cost", cl::Hidden, cl::init(false),
135 cl::desc("Add instruction count to a LSR cost model"));
136
Evgeny Stupachenko9909872e302017-02-21 07:34:40 +0000137// Flag to choose how to narrow complex lsr solution
138static cl::opt<bool> LSRExpNarrow(
Evgeny Stupachenkod6aa0d02017-03-04 03:14:05 +0000139 "lsr-exp-narrow", cl::Hidden, cl::init(false),
Evgeny Stupachenko9909872e302017-02-21 07:34:40 +0000140 cl::desc("Narrow LSR complex solution using"
141 " expectation of registers number"));
142
Andrew Trick248d4102012-01-09 21:18:52 +0000143#ifndef NDEBUG
144// Stress test IV chain generation.
145static cl::opt<bool> StressIVChain(
146 "stress-ivchain", cl::Hidden, cl::init(false),
147 cl::desc("Stress test LSR IV chains"));
148#else
149static bool StressIVChain = false;
150#endif
151
Dan Gohman45774ce2010-02-12 10:34:29 +0000152namespace {
Nate Begemanb18121e2004-10-18 21:08:22 +0000153
Matt Arsenault427a0fd2015-08-15 00:53:06 +0000154struct MemAccessTy {
155 /// Used in situations where the accessed memory type is unknown.
156 static const unsigned UnknownAddressSpace = ~0u;
157
158 Type *MemTy;
159 unsigned AddrSpace;
160
161 MemAccessTy() : MemTy(nullptr), AddrSpace(UnknownAddressSpace) {}
162
163 MemAccessTy(Type *Ty, unsigned AS) :
164 MemTy(Ty), AddrSpace(AS) {}
165
166 bool operator==(MemAccessTy Other) const {
167 return MemTy == Other.MemTy && AddrSpace == Other.AddrSpace;
168 }
169
170 bool operator!=(MemAccessTy Other) const { return !(*this == Other); }
171
Matt Arsenault1f2ca662017-01-30 19:50:17 +0000172 static MemAccessTy getUnknown(LLVMContext &Ctx,
173 unsigned AS = UnknownAddressSpace) {
174 return MemAccessTy(Type::getVoidTy(Ctx), AS);
Matt Arsenault427a0fd2015-08-15 00:53:06 +0000175 }
176};
177
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000178/// This class holds data which is used to order reuse candidates.
Dan Gohman45774ce2010-02-12 10:34:29 +0000179class RegSortData {
180public:
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000181 /// This represents the set of LSRUse indices which reference
Dan Gohman45774ce2010-02-12 10:34:29 +0000182 /// a particular register.
183 SmallBitVector UsedByIndices;
184
Dan Gohman45774ce2010-02-12 10:34:29 +0000185 void print(raw_ostream &OS) const;
186 void dump() const;
187};
188
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +0000189} // end anonymous namespace
Dan Gohman45774ce2010-02-12 10:34:29 +0000190
191void RegSortData::print(raw_ostream &OS) const {
192 OS << "[NumUses=" << UsedByIndices.count() << ']';
193}
194
Matthias Braun8c209aa2017-01-28 02:02:38 +0000195#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
196LLVM_DUMP_METHOD void RegSortData::dump() const {
Dan Gohman45774ce2010-02-12 10:34:29 +0000197 print(errs()); errs() << '\n';
198}
Matthias Braun8c209aa2017-01-28 02:02:38 +0000199#endif
Dan Gohman2a12ae72009-02-20 04:17:46 +0000200
Chris Lattner79a42ac2006-12-19 21:40:18 +0000201namespace {
Dale Johannesene3a02be2007-03-20 00:47:50 +0000202
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000203/// Map register candidates to information about how they are used.
Dan Gohman45774ce2010-02-12 10:34:29 +0000204class RegUseTracker {
205 typedef DenseMap<const SCEV *, RegSortData> RegUsesTy;
Dale Johannesene3a02be2007-03-20 00:47:50 +0000206
Dan Gohman248c41d2010-05-18 22:33:00 +0000207 RegUsesTy RegUsesMap;
Dan Gohman45774ce2010-02-12 10:34:29 +0000208 SmallVector<const SCEV *, 16> RegSequence;
Evan Cheng3df447d2006-03-16 21:53:05 +0000209
Dan Gohman45774ce2010-02-12 10:34:29 +0000210public:
Sanjoy Das302bfd02015-08-16 18:22:43 +0000211 void countRegister(const SCEV *Reg, size_t LUIdx);
212 void dropRegister(const SCEV *Reg, size_t LUIdx);
213 void swapAndDropUse(size_t LUIdx, size_t LastLUIdx);
Dan Gohman51ad99d2010-01-21 02:09:26 +0000214
Dan Gohman45774ce2010-02-12 10:34:29 +0000215 bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
Dan Gohman51ad99d2010-01-21 02:09:26 +0000216
Dan Gohman45774ce2010-02-12 10:34:29 +0000217 const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
Dan Gohman51ad99d2010-01-21 02:09:26 +0000218
Dan Gohman45774ce2010-02-12 10:34:29 +0000219 void clear();
Dan Gohman51ad99d2010-01-21 02:09:26 +0000220
Dan Gohman45774ce2010-02-12 10:34:29 +0000221 typedef SmallVectorImpl<const SCEV *>::iterator iterator;
222 typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator;
223 iterator begin() { return RegSequence.begin(); }
224 iterator end() { return RegSequence.end(); }
225 const_iterator begin() const { return RegSequence.begin(); }
226 const_iterator end() const { return RegSequence.end(); }
227};
Dan Gohman51ad99d2010-01-21 02:09:26 +0000228
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +0000229} // end anonymous namespace
Dan Gohman51ad99d2010-01-21 02:09:26 +0000230
Dan Gohman45774ce2010-02-12 10:34:29 +0000231void
Sanjoy Das302bfd02015-08-16 18:22:43 +0000232RegUseTracker::countRegister(const SCEV *Reg, size_t LUIdx) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000233 std::pair<RegUsesTy::iterator, bool> Pair =
Dan Gohman248c41d2010-05-18 22:33:00 +0000234 RegUsesMap.insert(std::make_pair(Reg, RegSortData()));
Dan Gohman45774ce2010-02-12 10:34:29 +0000235 RegSortData &RSD = Pair.first->second;
236 if (Pair.second)
237 RegSequence.push_back(Reg);
238 RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1));
239 RSD.UsedByIndices.set(LUIdx);
Dan Gohman51ad99d2010-01-21 02:09:26 +0000240}
241
Dan Gohman4cf99b52010-05-18 23:42:37 +0000242void
Sanjoy Das302bfd02015-08-16 18:22:43 +0000243RegUseTracker::dropRegister(const SCEV *Reg, size_t LUIdx) {
Dan Gohman4cf99b52010-05-18 23:42:37 +0000244 RegUsesTy::iterator It = RegUsesMap.find(Reg);
245 assert(It != RegUsesMap.end());
246 RegSortData &RSD = It->second;
247 assert(RSD.UsedByIndices.size() > LUIdx);
248 RSD.UsedByIndices.reset(LUIdx);
249}
250
Dan Gohman20fab452010-05-19 23:43:12 +0000251void
Sanjoy Das302bfd02015-08-16 18:22:43 +0000252RegUseTracker::swapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
Dan Gohmana7b68d62010-10-07 23:33:43 +0000253 assert(LUIdx <= LastLUIdx);
254
255 // Update RegUses. The data structure is not optimized for this purpose;
256 // we must iterate through it and update each of the bit vectors.
Craig Topper10949ae2015-05-23 08:45:10 +0000257 for (auto &Pair : RegUsesMap) {
258 SmallBitVector &UsedByIndices = Pair.second.UsedByIndices;
Dan Gohmana7b68d62010-10-07 23:33:43 +0000259 if (LUIdx < UsedByIndices.size())
260 UsedByIndices[LUIdx] =
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +0000261 LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : false;
Dan Gohmana7b68d62010-10-07 23:33:43 +0000262 UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx));
263 }
Dan Gohman20fab452010-05-19 23:43:12 +0000264}
265
Dan Gohman45774ce2010-02-12 10:34:29 +0000266bool
267RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const {
Dan Gohman4f13bbf2010-08-29 15:18:49 +0000268 RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
269 if (I == RegUsesMap.end())
270 return false;
271 const SmallBitVector &UsedByIndices = I->second.UsedByIndices;
Dan Gohman45774ce2010-02-12 10:34:29 +0000272 int i = UsedByIndices.find_first();
273 if (i == -1) return false;
274 if ((size_t)i != LUIdx) return true;
275 return UsedByIndices.find_next(i) != -1;
276}
Dan Gohman51ad99d2010-01-21 02:09:26 +0000277
Dan Gohman45774ce2010-02-12 10:34:29 +0000278const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const {
Dan Gohman248c41d2010-05-18 22:33:00 +0000279 RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
280 assert(I != RegUsesMap.end() && "Unknown register!");
Dan Gohman45774ce2010-02-12 10:34:29 +0000281 return I->second.UsedByIndices;
282}
Dan Gohman51ad99d2010-01-21 02:09:26 +0000283
Dan Gohman45774ce2010-02-12 10:34:29 +0000284void RegUseTracker::clear() {
Dan Gohman248c41d2010-05-18 22:33:00 +0000285 RegUsesMap.clear();
Dan Gohman45774ce2010-02-12 10:34:29 +0000286 RegSequence.clear();
287}
Dan Gohman51ad99d2010-01-21 02:09:26 +0000288
Dan Gohman45774ce2010-02-12 10:34:29 +0000289namespace {
290
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000291/// This class holds information that describes a formula for computing
292/// satisfying a use. It may include broken-out immediates and scaled registers.
Dan Gohman45774ce2010-02-12 10:34:29 +0000293struct Formula {
Chandler Carruth6e479322013-01-07 15:04:40 +0000294 /// Global base address used for complex addressing.
295 GlobalValue *BaseGV;
296
297 /// Base offset for complex addressing.
298 int64_t BaseOffset;
299
300 /// Whether any complex addressing has a base register.
301 bool HasBaseReg;
302
303 /// The scale of any complex addressing.
304 int64_t Scale;
Dan Gohman45774ce2010-02-12 10:34:29 +0000305
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000306 /// The list of "base" registers for this use. When this is non-empty. The
307 /// canonical representation of a formula is
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000308 /// 1. BaseRegs.size > 1 implies ScaledReg != NULL and
309 /// 2. ScaledReg != NULL implies Scale != 1 || !BaseRegs.empty().
Wei Mi74d5a902017-02-22 21:47:08 +0000310 /// 3. The reg containing recurrent expr related with currect loop in the
311 /// formula should be put in the ScaledReg.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000312 /// #1 enforces that the scaled register is always used when at least two
313 /// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 * reg2.
314 /// #2 enforces that 1 * reg is reg.
Wei Mi74d5a902017-02-22 21:47:08 +0000315 /// #3 ensures invariant regs with respect to current loop can be combined
316 /// together in LSR codegen.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000317 /// This invariant can be temporarly broken while building a formula.
318 /// However, every formula inserted into the LSRInstance must be in canonical
319 /// form.
Preston Gurd25c3b6a2013-02-01 20:41:27 +0000320 SmallVector<const SCEV *, 4> BaseRegs;
Dan Gohman45774ce2010-02-12 10:34:29 +0000321
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000322 /// The 'scaled' register for this use. This should be non-null when Scale is
323 /// not zero.
Dan Gohman45774ce2010-02-12 10:34:29 +0000324 const SCEV *ScaledReg;
325
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000326 /// An additional constant offset which added near the use. This requires a
327 /// temporary register, but the offset itself can live in an add immediate
328 /// field rather than a register.
Dan Gohman6136e942011-05-03 00:46:49 +0000329 int64_t UnfoldedOffset;
330
Chandler Carruth6e479322013-01-07 15:04:40 +0000331 Formula()
Craig Topperf40110f2014-04-25 05:29:35 +0000332 : BaseGV(nullptr), BaseOffset(0), HasBaseReg(false), Scale(0),
Sanjoy Das215df9e2015-08-04 01:52:05 +0000333 ScaledReg(nullptr), UnfoldedOffset(0) {}
Dan Gohman45774ce2010-02-12 10:34:29 +0000334
Sanjoy Das302bfd02015-08-16 18:22:43 +0000335 void initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
Dan Gohman45774ce2010-02-12 10:34:29 +0000336
Wei Mi74d5a902017-02-22 21:47:08 +0000337 bool isCanonical(const Loop &L) const;
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000338
Wei Mi74d5a902017-02-22 21:47:08 +0000339 void canonicalize(const Loop &L);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000340
Sanjoy Das302bfd02015-08-16 18:22:43 +0000341 bool unscale();
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000342
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +0000343 bool hasZeroEnd() const;
344
Adam Nemetdeab6f92014-04-29 18:25:28 +0000345 size_t getNumRegs() const;
Chris Lattner229907c2011-07-18 04:54:35 +0000346 Type *getType() const;
Dan Gohman45774ce2010-02-12 10:34:29 +0000347
Sanjoy Das302bfd02015-08-16 18:22:43 +0000348 void deleteBaseReg(const SCEV *&S);
Dan Gohman80a96082010-05-20 15:17:54 +0000349
Dan Gohman45774ce2010-02-12 10:34:29 +0000350 bool referencesReg(const SCEV *S) const;
351 bool hasRegsUsedByUsesOtherThan(size_t LUIdx,
352 const RegUseTracker &RegUses) const;
353
354 void print(raw_ostream &OS) const;
355 void dump() const;
356};
357
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +0000358} // end anonymous namespace
Dan Gohman45774ce2010-02-12 10:34:29 +0000359
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000360/// Recursion helper for initialMatch.
Dan Gohman45774ce2010-02-12 10:34:29 +0000361static void DoInitialMatch(const SCEV *S, Loop *L,
362 SmallVectorImpl<const SCEV *> &Good,
363 SmallVectorImpl<const SCEV *> &Bad,
Dan Gohman20d9ce22010-11-17 21:41:58 +0000364 ScalarEvolution &SE) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000365 // Collect expressions which properly dominate the loop header.
Dan Gohman20d9ce22010-11-17 21:41:58 +0000366 if (SE.properlyDominates(S, L->getHeader())) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000367 Good.push_back(S);
368 return;
Dan Gohman51ad99d2010-01-21 02:09:26 +0000369 }
Dan Gohman45774ce2010-02-12 10:34:29 +0000370
371 // Look at add operands.
372 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
Craig Topper77b99412015-05-23 08:01:41 +0000373 for (const SCEV *S : Add->operands())
374 DoInitialMatch(S, L, Good, Bad, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +0000375 return;
376 }
377
378 // Look at addrec operands.
379 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
Alexandros Lamprineas0ee3ec22016-11-09 08:53:07 +0000380 if (!AR->getStart()->isZero() && AR->isAffine()) {
Dan Gohman20d9ce22010-11-17 21:41:58 +0000381 DoInitialMatch(AR->getStart(), L, Good, Bad, SE);
Dan Gohman1d2ded72010-05-03 22:09:21 +0000382 DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0),
Dan Gohman45774ce2010-02-12 10:34:29 +0000383 AR->getStepRecurrence(SE),
Andrew Trick8b55b732011-03-14 16:50:06 +0000384 // FIXME: AR->getNoWrapFlags()
385 AR->getLoop(), SCEV::FlagAnyWrap),
Dan Gohman20d9ce22010-11-17 21:41:58 +0000386 L, Good, Bad, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +0000387 return;
388 }
389
390 // Handle a multiplication by -1 (negation) if it didn't fold.
391 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S))
392 if (Mul->getOperand(0)->isAllOnesValue()) {
393 SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end());
394 const SCEV *NewMul = SE.getMulExpr(Ops);
395
396 SmallVector<const SCEV *, 4> MyGood;
397 SmallVector<const SCEV *, 4> MyBad;
Dan Gohman20d9ce22010-11-17 21:41:58 +0000398 DoInitialMatch(NewMul, L, MyGood, MyBad, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +0000399 const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue(
400 SE.getEffectiveSCEVType(NewMul->getType())));
Craig Topper042a3922015-05-25 20:01:18 +0000401 for (const SCEV *S : MyGood)
402 Good.push_back(SE.getMulExpr(NegOne, S));
403 for (const SCEV *S : MyBad)
404 Bad.push_back(SE.getMulExpr(NegOne, S));
Dan Gohman45774ce2010-02-12 10:34:29 +0000405 return;
406 }
407
408 // Ok, we can't do anything interesting. Just stuff the whole thing into a
409 // register and hope for the best.
410 Bad.push_back(S);
411}
412
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000413/// Incorporate loop-variant parts of S into this Formula, attempting to keep
414/// all loop-invariant and loop-computable values in a single base register.
Sanjoy Das302bfd02015-08-16 18:22:43 +0000415void Formula::initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000416 SmallVector<const SCEV *, 4> Good;
417 SmallVector<const SCEV *, 4> Bad;
Dan Gohman20d9ce22010-11-17 21:41:58 +0000418 DoInitialMatch(S, L, Good, Bad, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +0000419 if (!Good.empty()) {
Dan Gohman9b5d0bb72010-04-08 23:36:27 +0000420 const SCEV *Sum = SE.getAddExpr(Good);
421 if (!Sum->isZero())
422 BaseRegs.push_back(Sum);
Chandler Carruth6e479322013-01-07 15:04:40 +0000423 HasBaseReg = true;
Dan Gohman45774ce2010-02-12 10:34:29 +0000424 }
425 if (!Bad.empty()) {
Dan Gohman9b5d0bb72010-04-08 23:36:27 +0000426 const SCEV *Sum = SE.getAddExpr(Bad);
427 if (!Sum->isZero())
428 BaseRegs.push_back(Sum);
Chandler Carruth6e479322013-01-07 15:04:40 +0000429 HasBaseReg = true;
Dan Gohman45774ce2010-02-12 10:34:29 +0000430 }
Wei Mi74d5a902017-02-22 21:47:08 +0000431 canonicalize(*L);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000432}
433
434/// \brief Check whether or not this formula statisfies the canonical
435/// representation.
436/// \see Formula::BaseRegs.
Wei Mi74d5a902017-02-22 21:47:08 +0000437bool Formula::isCanonical(const Loop &L) const {
438 if (!ScaledReg)
439 return BaseRegs.size() <= 1;
440
441 if (Scale != 1)
442 return true;
443
444 if (Scale == 1 && BaseRegs.empty())
445 return false;
446
447 const SCEVAddRecExpr *SAR = dyn_cast<const SCEVAddRecExpr>(ScaledReg);
448 if (SAR && SAR->getLoop() == &L)
449 return true;
450
451 // If ScaledReg is not a recurrent expr, or it is but its loop is not current
452 // loop, meanwhile BaseRegs contains a recurrent expr reg related with current
453 // loop, we want to swap the reg in BaseRegs with ScaledReg.
454 auto I =
455 find_if(make_range(BaseRegs.begin(), BaseRegs.end()), [&](const SCEV *S) {
456 return isa<const SCEVAddRecExpr>(S) &&
457 (cast<SCEVAddRecExpr>(S)->getLoop() == &L);
458 });
459 return I == BaseRegs.end();
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000460}
461
462/// \brief Helper method to morph a formula into its canonical representation.
463/// \see Formula::BaseRegs.
464/// Every formula having more than one base register, must use the ScaledReg
465/// field. Otherwise, we would have to do special cases everywhere in LSR
466/// to treat reg1 + reg2 + ... the same way as reg1 + 1*reg2 + ...
467/// On the other hand, 1*reg should be canonicalized into reg.
Wei Mi74d5a902017-02-22 21:47:08 +0000468void Formula::canonicalize(const Loop &L) {
469 if (isCanonical(L))
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000470 return;
471 // So far we did not need this case. This is easy to implement but it is
472 // useless to maintain dead code. Beside it could hurt compile time.
473 assert(!BaseRegs.empty() && "1*reg => reg, should not be needed.");
Wei Mi74d5a902017-02-22 21:47:08 +0000474
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000475 // Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
Wei Mi74d5a902017-02-22 21:47:08 +0000476 if (!ScaledReg) {
477 ScaledReg = BaseRegs.back();
478 BaseRegs.pop_back();
479 Scale = 1;
480 }
481
482 // If ScaledReg is an invariant with respect to L, find the reg from
483 // BaseRegs containing the recurrent expr related with Loop L. Swap the
484 // reg with ScaledReg.
485 const SCEVAddRecExpr *SAR = dyn_cast<const SCEVAddRecExpr>(ScaledReg);
486 if (!SAR || SAR->getLoop() != &L) {
487 auto I = find_if(make_range(BaseRegs.begin(), BaseRegs.end()),
488 [&](const SCEV *S) {
489 return isa<const SCEVAddRecExpr>(S) &&
490 (cast<SCEVAddRecExpr>(S)->getLoop() == &L);
491 });
492 if (I != BaseRegs.end())
493 std::swap(ScaledReg, *I);
494 }
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000495}
496
497/// \brief Get rid of the scale in the formula.
498/// In other words, this method morphes reg1 + 1*reg2 into reg1 + reg2.
499/// \return true if it was possible to get rid of the scale, false otherwise.
500/// \note After this operation the formula may not be in the canonical form.
Sanjoy Das302bfd02015-08-16 18:22:43 +0000501bool Formula::unscale() {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000502 if (Scale != 1)
503 return false;
504 Scale = 0;
505 BaseRegs.push_back(ScaledReg);
506 ScaledReg = nullptr;
507 return true;
Dan Gohman45774ce2010-02-12 10:34:29 +0000508}
509
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +0000510bool Formula::hasZeroEnd() const {
511 if (UnfoldedOffset || BaseOffset)
512 return false;
513 if (BaseRegs.size() != 1 || ScaledReg)
514 return false;
515 return true;
516}
517
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000518/// Return the total number of register operands used by this formula. This does
519/// not include register uses implied by non-constant addrec strides.
Adam Nemetdeab6f92014-04-29 18:25:28 +0000520size_t Formula::getNumRegs() const {
Dan Gohman45774ce2010-02-12 10:34:29 +0000521 return !!ScaledReg + BaseRegs.size();
522}
523
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000524/// Return the type of this formula, if it has one, or null otherwise. This type
525/// is meaningless except for the bit size.
Chris Lattner229907c2011-07-18 04:54:35 +0000526Type *Formula::getType() const {
Sanjoy Das215df9e2015-08-04 01:52:05 +0000527 return !BaseRegs.empty() ? BaseRegs.front()->getType() :
528 ScaledReg ? ScaledReg->getType() :
529 BaseGV ? BaseGV->getType() :
530 nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +0000531}
532
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000533/// Delete the given base reg from the BaseRegs list.
Sanjoy Das302bfd02015-08-16 18:22:43 +0000534void Formula::deleteBaseReg(const SCEV *&S) {
Dan Gohman80a96082010-05-20 15:17:54 +0000535 if (&S != &BaseRegs.back())
536 std::swap(S, BaseRegs.back());
537 BaseRegs.pop_back();
538}
539
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000540/// Test if this formula references the given register.
Dan Gohman45774ce2010-02-12 10:34:29 +0000541bool Formula::referencesReg(const SCEV *S) const {
David Majnemer0d955d02016-08-11 22:21:41 +0000542 return S == ScaledReg || is_contained(BaseRegs, S);
Dan Gohman45774ce2010-02-12 10:34:29 +0000543}
544
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000545/// Test whether this formula uses registers which are used by uses other than
546/// the use with the given index.
Dan Gohman45774ce2010-02-12 10:34:29 +0000547bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
548 const RegUseTracker &RegUses) const {
549 if (ScaledReg)
550 if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx))
551 return true;
Craig Topper042a3922015-05-25 20:01:18 +0000552 for (const SCEV *BaseReg : BaseRegs)
553 if (RegUses.isRegUsedByUsesOtherThan(BaseReg, LUIdx))
Dan Gohman45774ce2010-02-12 10:34:29 +0000554 return true;
555 return false;
556}
557
558void Formula::print(raw_ostream &OS) const {
559 bool First = true;
Chandler Carruth6e479322013-01-07 15:04:40 +0000560 if (BaseGV) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000561 if (!First) OS << " + "; else First = false;
Chandler Carruthd48cdbf2014-01-09 02:29:41 +0000562 BaseGV->printAsOperand(OS, /*PrintType=*/false);
Dan Gohman45774ce2010-02-12 10:34:29 +0000563 }
Chandler Carruth6e479322013-01-07 15:04:40 +0000564 if (BaseOffset != 0) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000565 if (!First) OS << " + "; else First = false;
Chandler Carruth6e479322013-01-07 15:04:40 +0000566 OS << BaseOffset;
Dan Gohman45774ce2010-02-12 10:34:29 +0000567 }
Craig Topper042a3922015-05-25 20:01:18 +0000568 for (const SCEV *BaseReg : BaseRegs) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000569 if (!First) OS << " + "; else First = false;
Sanjoy Das215df9e2015-08-04 01:52:05 +0000570 OS << "reg(" << *BaseReg << ')';
Dan Gohman45774ce2010-02-12 10:34:29 +0000571 }
Chandler Carruth6e479322013-01-07 15:04:40 +0000572 if (HasBaseReg && BaseRegs.empty()) {
Dan Gohman06ab08f2010-05-18 22:35:55 +0000573 if (!First) OS << " + "; else First = false;
574 OS << "**error: HasBaseReg**";
Chandler Carruth6e479322013-01-07 15:04:40 +0000575 } else if (!HasBaseReg && !BaseRegs.empty()) {
Dan Gohman06ab08f2010-05-18 22:35:55 +0000576 if (!First) OS << " + "; else First = false;
577 OS << "**error: !HasBaseReg**";
578 }
Chandler Carruth6e479322013-01-07 15:04:40 +0000579 if (Scale != 0) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000580 if (!First) OS << " + "; else First = false;
Chandler Carruth6e479322013-01-07 15:04:40 +0000581 OS << Scale << "*reg(";
Sanjoy Das215df9e2015-08-04 01:52:05 +0000582 if (ScaledReg)
583 OS << *ScaledReg;
584 else
Dan Gohman45774ce2010-02-12 10:34:29 +0000585 OS << "<unknown>";
586 OS << ')';
587 }
Dan Gohman6136e942011-05-03 00:46:49 +0000588 if (UnfoldedOffset != 0) {
Arnaud A. de Grandmaison75c9e6d2014-03-15 22:13:15 +0000589 if (!First) OS << " + ";
Dan Gohman6136e942011-05-03 00:46:49 +0000590 OS << "imm(" << UnfoldedOffset << ')';
591 }
Dan Gohman45774ce2010-02-12 10:34:29 +0000592}
593
Matthias Braun8c209aa2017-01-28 02:02:38 +0000594#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
595LLVM_DUMP_METHOD void Formula::dump() const {
Dan Gohman45774ce2010-02-12 10:34:29 +0000596 print(errs()); errs() << '\n';
597}
Matthias Braun8c209aa2017-01-28 02:02:38 +0000598#endif
Dan Gohman45774ce2010-02-12 10:34:29 +0000599
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000600/// Return true if the given addrec can be sign-extended without changing its
601/// value.
Dan Gohman85af2562010-02-19 19:32:49 +0000602static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
Chris Lattner229907c2011-07-18 04:54:35 +0000603 Type *WideTy =
Dan Gohmanab5fb7f2010-05-20 19:44:23 +0000604 IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1);
Dan Gohman85af2562010-02-19 19:32:49 +0000605 return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
606}
607
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000608/// Return true if the given add can be sign-extended without changing its
609/// value.
Dan Gohman85af2562010-02-19 19:32:49 +0000610static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
Chris Lattner229907c2011-07-18 04:54:35 +0000611 Type *WideTy =
Dan Gohmanab5fb7f2010-05-20 19:44:23 +0000612 IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1);
Dan Gohman85af2562010-02-19 19:32:49 +0000613 return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy));
614}
615
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000616/// Return true if the given mul can be sign-extended without changing its
617/// value.
Dan Gohmanab542222010-06-24 16:45:11 +0000618static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
Chris Lattner229907c2011-07-18 04:54:35 +0000619 Type *WideTy =
Dan Gohmanab542222010-06-24 16:45:11 +0000620 IntegerType::get(SE.getContext(),
621 SE.getTypeSizeInBits(M->getType()) * M->getNumOperands());
622 return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy));
Dan Gohman85af2562010-02-19 19:32:49 +0000623}
624
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000625/// Return an expression for LHS /s RHS, if it can be determined and if the
626/// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits
627/// is true, expressions like (X * Y) /s Y are simplified to Y, ignoring that
628/// the multiplication may overflow, which is useful when the result will be
629/// used in a context where the most significant bits are ignored.
Dan Gohman4eebb942010-02-19 19:35:48 +0000630static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
631 ScalarEvolution &SE,
632 bool IgnoreSignificantBits = false) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000633 // Handle the trivial case, which works for any SCEV type.
634 if (LHS == RHS)
Dan Gohman1d2ded72010-05-03 22:09:21 +0000635 return SE.getConstant(LHS->getType(), 1);
Dan Gohman45774ce2010-02-12 10:34:29 +0000636
Dan Gohman47ddf762010-06-24 16:51:25 +0000637 // Handle a few RHS special cases.
638 const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS);
639 if (RC) {
Sanjoy Das0de2fec2015-12-17 20:28:46 +0000640 const APInt &RA = RC->getAPInt();
Dan Gohman47ddf762010-06-24 16:51:25 +0000641 // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do
642 // some folding.
643 if (RA.isAllOnesValue())
644 return SE.getMulExpr(LHS, RC);
645 // Handle x /s 1 as x.
646 if (RA == 1)
647 return LHS;
648 }
Dan Gohman45774ce2010-02-12 10:34:29 +0000649
650 // Check for a division of a constant by a constant.
651 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000652 if (!RC)
Craig Topperf40110f2014-04-25 05:29:35 +0000653 return nullptr;
Sanjoy Das0de2fec2015-12-17 20:28:46 +0000654 const APInt &LA = C->getAPInt();
655 const APInt &RA = RC->getAPInt();
Dan Gohman47ddf762010-06-24 16:51:25 +0000656 if (LA.srem(RA) != 0)
Craig Topperf40110f2014-04-25 05:29:35 +0000657 return nullptr;
Dan Gohman47ddf762010-06-24 16:51:25 +0000658 return SE.getConstant(LA.sdiv(RA));
Dan Gohman45774ce2010-02-12 10:34:29 +0000659 }
660
Dan Gohman85af2562010-02-19 19:32:49 +0000661 // Distribute the sdiv over addrec operands, if the addrec doesn't overflow.
Dan Gohman45774ce2010-02-12 10:34:29 +0000662 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) {
Alexandros Lamprineas0ee3ec22016-11-09 08:53:07 +0000663 if ((IgnoreSignificantBits || isAddRecSExtable(AR, SE)) && AR->isAffine()) {
Dan Gohman4eebb942010-02-19 19:35:48 +0000664 const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE,
665 IgnoreSignificantBits);
Craig Topperf40110f2014-04-25 05:29:35 +0000666 if (!Step) return nullptr;
Dan Gohman129a8162010-08-19 01:02:31 +0000667 const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE,
668 IgnoreSignificantBits);
Craig Topperf40110f2014-04-25 05:29:35 +0000669 if (!Start) return nullptr;
Andrew Trick8b55b732011-03-14 16:50:06 +0000670 // FlagNW is independent of the start value, step direction, and is
671 // preserved with smaller magnitude steps.
672 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
673 return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap);
Dan Gohman85af2562010-02-19 19:32:49 +0000674 }
Craig Topperf40110f2014-04-25 05:29:35 +0000675 return nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +0000676 }
677
Dan Gohman85af2562010-02-19 19:32:49 +0000678 // Distribute the sdiv over add operands, if the add doesn't overflow.
Dan Gohman45774ce2010-02-12 10:34:29 +0000679 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) {
Dan Gohman85af2562010-02-19 19:32:49 +0000680 if (IgnoreSignificantBits || isAddSExtable(Add, SE)) {
681 SmallVector<const SCEV *, 8> Ops;
Craig Topper042a3922015-05-25 20:01:18 +0000682 for (const SCEV *S : Add->operands()) {
683 const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits);
Craig Topperf40110f2014-04-25 05:29:35 +0000684 if (!Op) return nullptr;
Dan Gohman85af2562010-02-19 19:32:49 +0000685 Ops.push_back(Op);
686 }
687 return SE.getAddExpr(Ops);
Dan Gohman45774ce2010-02-12 10:34:29 +0000688 }
Craig Topperf40110f2014-04-25 05:29:35 +0000689 return nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +0000690 }
691
692 // Check for a multiply operand that we can pull RHS out of.
Dan Gohman963b1c12010-06-24 16:57:52 +0000693 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) {
Dan Gohman85af2562010-02-19 19:32:49 +0000694 if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000695 SmallVector<const SCEV *, 4> Ops;
696 bool Found = false;
Craig Topper042a3922015-05-25 20:01:18 +0000697 for (const SCEV *S : Mul->operands()) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000698 if (!Found)
Dan Gohman6b733fc2010-05-20 16:23:28 +0000699 if (const SCEV *Q = getExactSDiv(S, RHS, SE,
Dan Gohman4eebb942010-02-19 19:35:48 +0000700 IgnoreSignificantBits)) {
Dan Gohman6b733fc2010-05-20 16:23:28 +0000701 S = Q;
Dan Gohman45774ce2010-02-12 10:34:29 +0000702 Found = true;
Dan Gohman45774ce2010-02-12 10:34:29 +0000703 }
Dan Gohman6b733fc2010-05-20 16:23:28 +0000704 Ops.push_back(S);
Dan Gohman45774ce2010-02-12 10:34:29 +0000705 }
Craig Topperf40110f2014-04-25 05:29:35 +0000706 return Found ? SE.getMulExpr(Ops) : nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +0000707 }
Craig Topperf40110f2014-04-25 05:29:35 +0000708 return nullptr;
Dan Gohman963b1c12010-06-24 16:57:52 +0000709 }
Dan Gohman45774ce2010-02-12 10:34:29 +0000710
711 // Otherwise we don't know.
Craig Topperf40110f2014-04-25 05:29:35 +0000712 return nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +0000713}
714
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000715/// If S involves the addition of a constant integer value, return that integer
716/// value, and mutate S to point to a new SCEV with that value excluded.
Dan Gohman45774ce2010-02-12 10:34:29 +0000717static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
718 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
Sanjoy Das0de2fec2015-12-17 20:28:46 +0000719 if (C->getAPInt().getMinSignedBits() <= 64) {
Dan Gohman1d2ded72010-05-03 22:09:21 +0000720 S = SE.getConstant(C->getType(), 0);
Dan Gohman45774ce2010-02-12 10:34:29 +0000721 return C->getValue()->getSExtValue();
722 }
723 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
724 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end());
725 int64_t Result = ExtractImmediate(NewOps.front(), SE);
Dan Gohman081ffcd2010-08-13 21:17:19 +0000726 if (Result != 0)
727 S = SE.getAddExpr(NewOps);
Dan Gohman45774ce2010-02-12 10:34:29 +0000728 return Result;
729 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
730 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end());
731 int64_t Result = ExtractImmediate(NewOps.front(), SE);
Dan Gohman081ffcd2010-08-13 21:17:19 +0000732 if (Result != 0)
Andrew Trick8b55b732011-03-14 16:50:06 +0000733 S = SE.getAddRecExpr(NewOps, AR->getLoop(),
734 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
735 SCEV::FlagAnyWrap);
Dan Gohman45774ce2010-02-12 10:34:29 +0000736 return Result;
737 }
738 return 0;
739}
740
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000741/// If S involves the addition of a GlobalValue address, return that symbol, and
742/// mutate S to point to a new SCEV with that value excluded.
Dan Gohman45774ce2010-02-12 10:34:29 +0000743static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) {
744 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
745 if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) {
Dan Gohman1d2ded72010-05-03 22:09:21 +0000746 S = SE.getConstant(GV->getType(), 0);
Dan Gohman45774ce2010-02-12 10:34:29 +0000747 return GV;
748 }
749 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
750 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end());
751 GlobalValue *Result = ExtractSymbol(NewOps.back(), SE);
Dan Gohman081ffcd2010-08-13 21:17:19 +0000752 if (Result)
753 S = SE.getAddExpr(NewOps);
Dan Gohman45774ce2010-02-12 10:34:29 +0000754 return Result;
755 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
756 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end());
757 GlobalValue *Result = ExtractSymbol(NewOps.front(), SE);
Dan Gohman081ffcd2010-08-13 21:17:19 +0000758 if (Result)
Andrew Trick8b55b732011-03-14 16:50:06 +0000759 S = SE.getAddRecExpr(NewOps, AR->getLoop(),
760 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
761 SCEV::FlagAnyWrap);
Dan Gohman45774ce2010-02-12 10:34:29 +0000762 return Result;
763 }
Craig Topperf40110f2014-04-25 05:29:35 +0000764 return nullptr;
Nate Begemanb18121e2004-10-18 21:08:22 +0000765}
766
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000767/// Returns true if the specified instruction is using the specified value as an
768/// address.
Dale Johannesen9efd2ce2008-12-05 21:47:27 +0000769static bool isAddressUse(Instruction *Inst, Value *OperandVal) {
770 bool isAddress = isa<LoadInst>(Inst);
771 if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
Matt Arsenaultcb3fa372017-02-08 06:44:58 +0000772 if (SI->getPointerOperand() == OperandVal)
Dale Johannesen9efd2ce2008-12-05 21:47:27 +0000773 isAddress = true;
774 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
775 // Addressing modes can also be folded into prefetches and a variety
776 // of intrinsics.
777 switch (II->getIntrinsicID()) {
778 default: break;
779 case Intrinsic::prefetch:
Gabor Greif8ae30952010-06-30 09:15:28 +0000780 if (II->getArgOperand(0) == OperandVal)
Dale Johannesen9efd2ce2008-12-05 21:47:27 +0000781 isAddress = true;
782 break;
783 }
Matt Arsenaultcb3fa372017-02-08 06:44:58 +0000784 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) {
785 if (RMW->getPointerOperand() == OperandVal)
786 isAddress = true;
787 } else if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {
788 if (CmpX->getPointerOperand() == OperandVal)
789 isAddress = true;
Dale Johannesen9efd2ce2008-12-05 21:47:27 +0000790 }
791 return isAddress;
792}
Chris Lattnere4ed42a2005-10-03 01:04:44 +0000793
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000794/// Return the type of the memory being accessed.
Matt Arsenault427a0fd2015-08-15 00:53:06 +0000795static MemAccessTy getAccessType(const Instruction *Inst) {
796 MemAccessTy AccessTy(Inst->getType(), MemAccessTy::UnknownAddressSpace);
797 if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
798 AccessTy.MemTy = SI->getOperand(0)->getType();
799 AccessTy.AddrSpace = SI->getPointerAddressSpace();
800 } else if (const LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
801 AccessTy.AddrSpace = LI->getPointerAddressSpace();
Matt Arsenaultcb3fa372017-02-08 06:44:58 +0000802 } else if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) {
803 AccessTy.AddrSpace = RMW->getPointerAddressSpace();
804 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {
805 AccessTy.AddrSpace = CmpX->getPointerAddressSpace();
Dan Gohman917ffe42009-03-09 21:01:17 +0000806 }
Dan Gohman45774ce2010-02-12 10:34:29 +0000807
808 // All pointers have the same requirements, so canonicalize them to an
809 // arbitrary pointer type to minimize variation.
Matt Arsenault427a0fd2015-08-15 00:53:06 +0000810 if (PointerType *PTy = dyn_cast<PointerType>(AccessTy.MemTy))
811 AccessTy.MemTy = PointerType::get(IntegerType::get(PTy->getContext(), 1),
812 PTy->getAddressSpace());
Dan Gohman45774ce2010-02-12 10:34:29 +0000813
Dan Gohman14d13392009-05-18 16:45:28 +0000814 return AccessTy;
Dan Gohman917ffe42009-03-09 21:01:17 +0000815}
816
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000817/// Return true if this AddRec is already a phi in its loop.
Andrew Trick5df90962011-12-06 03:13:31 +0000818static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
819 for (BasicBlock::iterator I = AR->getLoop()->getHeader()->begin();
820 PHINode *PN = dyn_cast<PHINode>(I); ++I) {
821 if (SE.isSCEVable(PN->getType()) &&
822 (SE.getEffectiveSCEVType(PN->getType()) ==
823 SE.getEffectiveSCEVType(AR->getType())) &&
824 SE.getSCEV(PN) == AR)
825 return true;
826 }
827 return false;
828}
829
Andrew Trickd5d2db92012-01-10 01:45:08 +0000830/// Check if expanding this expression is likely to incur significant cost. This
831/// is tricky because SCEV doesn't track which expressions are actually computed
832/// by the current IR.
833///
834/// We currently allow expansion of IV increments that involve adds,
835/// multiplication by constants, and AddRecs from existing phis.
836///
837/// TODO: Allow UDivExpr if we can find an existing IV increment that is an
838/// obvious multiple of the UDivExpr.
839static bool isHighCostExpansion(const SCEV *S,
Craig Topper71b7b682014-08-21 05:55:13 +0000840 SmallPtrSetImpl<const SCEV*> &Processed,
Andrew Trickd5d2db92012-01-10 01:45:08 +0000841 ScalarEvolution &SE) {
842 // Zero/One operand expressions
843 switch (S->getSCEVType()) {
844 case scUnknown:
845 case scConstant:
846 return false;
847 case scTruncate:
848 return isHighCostExpansion(cast<SCEVTruncateExpr>(S)->getOperand(),
849 Processed, SE);
850 case scZeroExtend:
851 return isHighCostExpansion(cast<SCEVZeroExtendExpr>(S)->getOperand(),
852 Processed, SE);
853 case scSignExtend:
854 return isHighCostExpansion(cast<SCEVSignExtendExpr>(S)->getOperand(),
855 Processed, SE);
856 }
857
David Blaikie70573dc2014-11-19 07:49:26 +0000858 if (!Processed.insert(S).second)
Andrew Trickd5d2db92012-01-10 01:45:08 +0000859 return false;
860
861 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
Craig Topper042a3922015-05-25 20:01:18 +0000862 for (const SCEV *S : Add->operands()) {
863 if (isHighCostExpansion(S, Processed, SE))
Andrew Trickd5d2db92012-01-10 01:45:08 +0000864 return true;
865 }
866 return false;
867 }
868
869 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
870 if (Mul->getNumOperands() == 2) {
871 // Multiplication by a constant is ok
872 if (isa<SCEVConstant>(Mul->getOperand(0)))
873 return isHighCostExpansion(Mul->getOperand(1), Processed, SE);
874
875 // If we have the value of one operand, check if an existing
876 // multiplication already generates this expression.
877 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) {
878 Value *UVal = U->getValue();
Chandler Carruthcdf47882014-03-09 03:16:01 +0000879 for (User *UR : UVal->users()) {
Andrew Trick14779cc2012-03-26 20:28:37 +0000880 // If U is a constant, it may be used by a ConstantExpr.
Chandler Carruthcdf47882014-03-09 03:16:01 +0000881 Instruction *UI = dyn_cast<Instruction>(UR);
882 if (UI && UI->getOpcode() == Instruction::Mul &&
883 SE.isSCEVable(UI->getType())) {
884 return SE.getSCEV(UI) == Mul;
Andrew Trickd5d2db92012-01-10 01:45:08 +0000885 }
886 }
887 }
888 }
889 }
890
891 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
892 if (isExistingPhi(AR, SE))
893 return false;
894 }
895
896 // Fow now, consider any other type of expression (div/mul/min/max) high cost.
897 return true;
898}
899
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000900/// If any of the instructions is the specified set are trivially dead, delete
901/// them and see if this makes any of their operands subsequently dead.
Dan Gohman45774ce2010-02-12 10:34:29 +0000902static bool
Sanjoy Dase6bca0e2017-05-01 17:07:49 +0000903DeleteTriviallyDeadInstructions(SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
Dan Gohman45774ce2010-02-12 10:34:29 +0000904 bool Changed = false;
905
906 while (!DeadInsts.empty()) {
Richard Smithad9c8e82012-08-21 20:35:14 +0000907 Value *V = DeadInsts.pop_back_val();
908 Instruction *I = dyn_cast_or_null<Instruction>(V);
Dan Gohman45774ce2010-02-12 10:34:29 +0000909
Craig Topperf40110f2014-04-25 05:29:35 +0000910 if (!I || !isInstructionTriviallyDead(I))
Dan Gohman45774ce2010-02-12 10:34:29 +0000911 continue;
912
Craig Topper042a3922015-05-25 20:01:18 +0000913 for (Use &O : I->operands())
914 if (Instruction *U = dyn_cast<Instruction>(O)) {
915 O = nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +0000916 if (U->use_empty())
Benjamin Kramerf5e2fc42015-05-29 19:43:39 +0000917 DeadInsts.emplace_back(U);
Dan Gohman45774ce2010-02-12 10:34:29 +0000918 }
919
920 I->eraseFromParent();
921 Changed = true;
922 }
923
924 return Changed;
925}
926
Dan Gohman045f8192010-01-22 00:46:49 +0000927namespace {
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +0000928
Quentin Colombet8aa7abe2013-05-31 17:20:29 +0000929class LSRUse;
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +0000930
931} // end anonymous namespace
Quentin Colombetc88baa5c2014-05-20 19:25:04 +0000932
933/// \brief Check if the addressing mode defined by \p F is completely
934/// folded in \p LU at isel time.
935/// This includes address-mode folding and special icmp tricks.
936/// This function returns true if \p LU can accommodate what \p F
937/// defines and up to 1 base + 1 scaled + offset.
938/// In other words, if \p F has several base registers, this function may
939/// still return true. Therefore, users still need to account for
940/// additional base registers and/or unfolded offsets to derive an
941/// accurate cost model.
942static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
943 const LSRUse &LU, const Formula &F);
Quentin Colombetbf490d42013-05-31 21:29:03 +0000944// Get the cost of the scaling factor used in F for LU.
945static unsigned getScalingFactorCost(const TargetTransformInfo &TTI,
Wei Mi74d5a902017-02-22 21:47:08 +0000946 const LSRUse &LU, const Formula &F,
947 const Loop &L);
Quentin Colombet8aa7abe2013-05-31 17:20:29 +0000948
949namespace {
Jim Grosbach60f48542009-11-17 17:53:56 +0000950
Sanjoy Das94c4aec2015-08-16 18:22:46 +0000951/// This class is used to measure and compare candidate formulae.
Dan Gohman45774ce2010-02-12 10:34:29 +0000952class Cost {
953 /// TODO: Some of these could be merged. Also, a lexical ordering
954 /// isn't always optimal.
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +0000955 unsigned Insns;
Dan Gohman45774ce2010-02-12 10:34:29 +0000956 unsigned NumRegs;
957 unsigned AddRecCost;
958 unsigned NumIVMuls;
959 unsigned NumBaseAdds;
960 unsigned ImmCost;
961 unsigned SetupCost;
Quentin Colombetbf490d42013-05-31 21:29:03 +0000962 unsigned ScaleCost;
Nate Begemane68bcd12005-07-30 00:15:07 +0000963
Dan Gohman45774ce2010-02-12 10:34:29 +0000964public:
965 Cost()
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +0000966 : Insns(0), NumRegs(0), AddRecCost(0), NumIVMuls(0), NumBaseAdds(0),
967 ImmCost(0), SetupCost(0), ScaleCost(0) {}
Jim Grosbach60f48542009-11-17 17:53:56 +0000968
Dan Gohman45774ce2010-02-12 10:34:29 +0000969 bool operator<(const Cost &Other) const;
Dan Gohman045f8192010-01-22 00:46:49 +0000970
Tim Northoverbc6659c2014-01-22 13:27:00 +0000971 void Lose();
Dan Gohman045f8192010-01-22 00:46:49 +0000972
Andrew Trick784729d2011-09-26 23:11:04 +0000973#ifndef NDEBUG
974 // Once any of the metrics loses, they must all remain losers.
975 bool isValid() {
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +0000976 return ((Insns | NumRegs | AddRecCost | NumIVMuls | NumBaseAdds
Quentin Colombetbf490d42013-05-31 21:29:03 +0000977 | ImmCost | SetupCost | ScaleCost) != ~0u)
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +0000978 || ((Insns & NumRegs & AddRecCost & NumIVMuls & NumBaseAdds
Quentin Colombetbf490d42013-05-31 21:29:03 +0000979 & ImmCost & SetupCost & ScaleCost) == ~0u);
Andrew Trick784729d2011-09-26 23:11:04 +0000980 }
981#endif
982
983 bool isLoser() {
984 assert(isValid() && "invalid cost");
985 return NumRegs == ~0u;
986 }
987
Quentin Colombet8aa7abe2013-05-31 17:20:29 +0000988 void RateFormula(const TargetTransformInfo &TTI,
989 const Formula &F,
Craig Topper71b7b682014-08-21 05:55:13 +0000990 SmallPtrSetImpl<const SCEV *> &Regs,
Dan Gohman45774ce2010-02-12 10:34:29 +0000991 const DenseSet<const SCEV *> &VisitedRegs,
992 const Loop *L,
Andrew Trick5df90962011-12-06 03:13:31 +0000993 ScalarEvolution &SE, DominatorTree &DT,
Quentin Colombet8aa7abe2013-05-31 17:20:29 +0000994 const LSRUse &LU,
Craig Topper71b7b682014-08-21 05:55:13 +0000995 SmallPtrSetImpl<const SCEV *> *LoserRegs = nullptr);
Dan Gohman045f8192010-01-22 00:46:49 +0000996
Dan Gohman45774ce2010-02-12 10:34:29 +0000997 void print(raw_ostream &OS) const;
998 void dump() const;
Dan Gohman045f8192010-01-22 00:46:49 +0000999
Dan Gohman45774ce2010-02-12 10:34:29 +00001000private:
1001 void RateRegister(const SCEV *Reg,
Craig Topper71b7b682014-08-21 05:55:13 +00001002 SmallPtrSetImpl<const SCEV *> &Regs,
Dan Gohman45774ce2010-02-12 10:34:29 +00001003 const Loop *L,
1004 ScalarEvolution &SE, DominatorTree &DT);
Dan Gohman5b18f032010-02-13 02:06:02 +00001005 void RatePrimaryRegister(const SCEV *Reg,
Craig Topper71b7b682014-08-21 05:55:13 +00001006 SmallPtrSetImpl<const SCEV *> &Regs,
Dan Gohman5b18f032010-02-13 02:06:02 +00001007 const Loop *L,
Andrew Trick5df90962011-12-06 03:13:31 +00001008 ScalarEvolution &SE, DominatorTree &DT,
Craig Topper71b7b682014-08-21 05:55:13 +00001009 SmallPtrSetImpl<const SCEV *> *LoserRegs);
Dan Gohman45774ce2010-02-12 10:34:29 +00001010};
Jonas Paulsson7a794222016-08-17 13:24:19 +00001011
1012/// An operand value in an instruction which is to be replaced with some
1013/// equivalent, possibly strength-reduced, replacement.
1014struct LSRFixup {
1015 /// The instruction which will be updated.
1016 Instruction *UserInst;
1017
1018 /// The operand of the instruction which will be replaced. The operand may be
1019 /// used more than once; every instance will be replaced.
1020 Value *OperandValToReplace;
1021
1022 /// If this user is to use the post-incremented value of an induction
1023 /// variable, this variable is non-null and holds the loop associated with the
1024 /// induction variable.
1025 PostIncLoopSet PostIncLoops;
1026
1027 /// A constant offset to be added to the LSRUse expression. This allows
1028 /// multiple fixups to share the same LSRUse with different offsets, for
1029 /// example in an unrolled loop.
1030 int64_t Offset;
1031
1032 bool isUseFullyOutsideLoop(const Loop *L) const;
1033
1034 LSRFixup();
1035
1036 void print(raw_ostream &OS) const;
1037 void dump() const;
1038};
1039
Jonas Paulsson7a794222016-08-17 13:24:19 +00001040/// A DenseMapInfo implementation for holding DenseMaps and DenseSets of sorted
1041/// SmallVectors of const SCEV*.
1042struct UniquifierDenseMapInfo {
1043 static SmallVector<const SCEV *, 4> getEmptyKey() {
1044 SmallVector<const SCEV *, 4> V;
1045 V.push_back(reinterpret_cast<const SCEV *>(-1));
1046 return V;
1047 }
1048
1049 static SmallVector<const SCEV *, 4> getTombstoneKey() {
1050 SmallVector<const SCEV *, 4> V;
1051 V.push_back(reinterpret_cast<const SCEV *>(-2));
1052 return V;
1053 }
1054
1055 static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) {
1056 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1057 }
1058
1059 static bool isEqual(const SmallVector<const SCEV *, 4> &LHS,
1060 const SmallVector<const SCEV *, 4> &RHS) {
1061 return LHS == RHS;
1062 }
1063};
1064
1065/// This class holds the state that LSR keeps for each use in IVUsers, as well
1066/// as uses invented by LSR itself. It includes information about what kinds of
1067/// things can be folded into the user, information about the user itself, and
1068/// information about how the use may be satisfied. TODO: Represent multiple
1069/// users of the same expression in common?
1070class LSRUse {
1071 DenseSet<SmallVector<const SCEV *, 4>, UniquifierDenseMapInfo> Uniquifier;
1072
1073public:
1074 /// An enum for a kind of use, indicating what types of scaled and immediate
1075 /// operands it might support.
1076 enum KindType {
1077 Basic, ///< A normal use, with no folding.
1078 Special, ///< A special case of basic, allowing -1 scales.
1079 Address, ///< An address use; folding according to TargetLowering
1080 ICmpZero ///< An equality icmp with both operands folded into one.
1081 // TODO: Add a generic icmp too?
1082 };
1083
1084 typedef PointerIntPair<const SCEV *, 2, KindType> SCEVUseKindPair;
1085
1086 KindType Kind;
1087 MemAccessTy AccessTy;
1088
1089 /// The list of operands which are to be replaced.
1090 SmallVector<LSRFixup, 8> Fixups;
1091
1092 /// Keep track of the min and max offsets of the fixups.
1093 int64_t MinOffset;
1094 int64_t MaxOffset;
1095
1096 /// This records whether all of the fixups using this LSRUse are outside of
1097 /// the loop, in which case some special-case heuristics may be used.
1098 bool AllFixupsOutsideLoop;
1099
1100 /// RigidFormula is set to true to guarantee that this use will be associated
1101 /// with a single formula--the one that initially matched. Some SCEV
1102 /// expressions cannot be expanded. This allows LSR to consider the registers
1103 /// used by those expressions without the need to expand them later after
1104 /// changing the formula.
1105 bool RigidFormula;
1106
1107 /// This records the widest use type for any fixup using this
1108 /// LSRUse. FindUseWithSimilarFormula can't consider uses with different max
1109 /// fixup widths to be equivalent, because the narrower one may be relying on
1110 /// the implicit truncation to truncate away bogus bits.
1111 Type *WidestFixupType;
1112
1113 /// A list of ways to build a value that can satisfy this user. After the
1114 /// list is populated, one of these is selected heuristically and used to
1115 /// formulate a replacement for OperandValToReplace in UserInst.
1116 SmallVector<Formula, 12> Formulae;
1117
1118 /// The set of register candidates used by all formulae in this LSRUse.
1119 SmallPtrSet<const SCEV *, 4> Regs;
1120
1121 LSRUse(KindType K, MemAccessTy AT)
1122 : Kind(K), AccessTy(AT), MinOffset(INT64_MAX), MaxOffset(INT64_MIN),
1123 AllFixupsOutsideLoop(true), RigidFormula(false),
1124 WidestFixupType(nullptr) {}
1125
1126 LSRFixup &getNewFixup() {
1127 Fixups.push_back(LSRFixup());
1128 return Fixups.back();
1129 }
1130
1131 void pushFixup(LSRFixup &f) {
1132 Fixups.push_back(f);
1133 if (f.Offset > MaxOffset)
1134 MaxOffset = f.Offset;
1135 if (f.Offset < MinOffset)
1136 MinOffset = f.Offset;
1137 }
1138
1139 bool HasFormulaWithSameRegs(const Formula &F) const;
Evgeny Stupachenko9909872e302017-02-21 07:34:40 +00001140 float getNotSelectedProbability(const SCEV *Reg) const;
Wei Mi74d5a902017-02-22 21:47:08 +00001141 bool InsertFormula(const Formula &F, const Loop &L);
Jonas Paulsson7a794222016-08-17 13:24:19 +00001142 void DeleteFormula(Formula &F);
1143 void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses);
1144
1145 void print(raw_ostream &OS) const;
1146 void dump() const;
1147};
Dan Gohman45774ce2010-02-12 10:34:29 +00001148
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00001149} // end anonymous namespace
Dan Gohman45774ce2010-02-12 10:34:29 +00001150
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001151/// Tally up interesting quantities from the given register.
Dan Gohman45774ce2010-02-12 10:34:29 +00001152void Cost::RateRegister(const SCEV *Reg,
Craig Topper71b7b682014-08-21 05:55:13 +00001153 SmallPtrSetImpl<const SCEV *> &Regs,
Dan Gohman45774ce2010-02-12 10:34:29 +00001154 const Loop *L,
1155 ScalarEvolution &SE, DominatorTree &DT) {
Dan Gohman5b18f032010-02-13 02:06:02 +00001156 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) {
Wei Mi8f20e632017-02-11 00:50:23 +00001157 // If this is an addrec for another loop, it should be an invariant
1158 // with respect to L since L is the innermost loop (at least
1159 // for now LSR only handles innermost loops).
Andrew Trickd97b83e2012-03-22 22:42:45 +00001160 if (AR->getLoop() != L) {
1161 // If the AddRec exists, consider it's register free and leave it alone.
Andrew Trick5df90962011-12-06 03:13:31 +00001162 if (isExistingPhi(AR, SE))
1163 return;
1164
Wei Mi493fb262017-02-16 21:27:31 +00001165 // It is bad to allow LSR for current loop to add induction variables
1166 // for its sibling loops.
1167 if (!AR->getLoop()->contains(L)) {
1168 Lose();
1169 return;
1170 }
1171
Wei Mi8f20e632017-02-11 00:50:23 +00001172 // Otherwise, it will be an invariant with respect to Loop L.
1173 ++NumRegs;
Andrew Trickd97b83e2012-03-22 22:42:45 +00001174 return;
Dan Gohman45774ce2010-02-12 10:34:29 +00001175 }
Andrew Trickd97b83e2012-03-22 22:42:45 +00001176 AddRecCost += 1; /// TODO: This should be a function of the stride.
Dan Gohman45774ce2010-02-12 10:34:29 +00001177
Dan Gohman5b18f032010-02-13 02:06:02 +00001178 // Add the step value register, if it needs one.
1179 // TODO: The non-affine case isn't precisely modeled here.
Andrew Trick8868fae2011-09-26 23:35:25 +00001180 if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) {
1181 if (!Regs.count(AR->getOperand(1))) {
Dan Gohman5b18f032010-02-13 02:06:02 +00001182 RateRegister(AR->getOperand(1), Regs, L, SE, DT);
Andrew Trick8868fae2011-09-26 23:35:25 +00001183 if (isLoser())
1184 return;
1185 }
1186 }
Dan Gohman45774ce2010-02-12 10:34:29 +00001187 }
Dan Gohman5b18f032010-02-13 02:06:02 +00001188 ++NumRegs;
1189
1190 // Rough heuristic; favor registers which don't require extra setup
1191 // instructions in the preheader.
1192 if (!isa<SCEVUnknown>(Reg) &&
1193 !isa<SCEVConstant>(Reg) &&
1194 !(isa<SCEVAddRecExpr>(Reg) &&
1195 (isa<SCEVUnknown>(cast<SCEVAddRecExpr>(Reg)->getStart()) ||
1196 isa<SCEVConstant>(cast<SCEVAddRecExpr>(Reg)->getStart()))))
1197 ++SetupCost;
Dan Gohman34f37e02010-10-07 23:41:58 +00001198
Davide Italiano709d4182016-07-07 17:44:38 +00001199 NumIVMuls += isa<SCEVMulExpr>(Reg) &&
1200 SE.hasComputableLoopEvolution(Reg, L);
Dan Gohman5b18f032010-02-13 02:06:02 +00001201}
1202
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001203/// Record this register in the set. If we haven't seen it before, rate
1204/// it. Optional LoserRegs provides a way to declare any formula that refers to
1205/// one of those regs an instant loser.
Dan Gohman5b18f032010-02-13 02:06:02 +00001206void Cost::RatePrimaryRegister(const SCEV *Reg,
Craig Topper71b7b682014-08-21 05:55:13 +00001207 SmallPtrSetImpl<const SCEV *> &Regs,
Dan Gohman0849ed52010-02-16 19:42:34 +00001208 const Loop *L,
Andrew Trick5df90962011-12-06 03:13:31 +00001209 ScalarEvolution &SE, DominatorTree &DT,
Craig Topper71b7b682014-08-21 05:55:13 +00001210 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
Andrew Trick5df90962011-12-06 03:13:31 +00001211 if (LoserRegs && LoserRegs->count(Reg)) {
Tim Northoverbc6659c2014-01-22 13:27:00 +00001212 Lose();
Andrew Trick5df90962011-12-06 03:13:31 +00001213 return;
1214 }
David Blaikie70573dc2014-11-19 07:49:26 +00001215 if (Regs.insert(Reg).second) {
Dan Gohman5b18f032010-02-13 02:06:02 +00001216 RateRegister(Reg, Regs, L, SE, DT);
Andrew Tricka1c01ba2013-03-19 04:14:57 +00001217 if (LoserRegs && isLoser())
Andrew Trick5df90962011-12-06 03:13:31 +00001218 LoserRegs->insert(Reg);
1219 }
Dan Gohman45774ce2010-02-12 10:34:29 +00001220}
1221
Quentin Colombet8aa7abe2013-05-31 17:20:29 +00001222void Cost::RateFormula(const TargetTransformInfo &TTI,
1223 const Formula &F,
Craig Topper71b7b682014-08-21 05:55:13 +00001224 SmallPtrSetImpl<const SCEV *> &Regs,
Dan Gohman45774ce2010-02-12 10:34:29 +00001225 const DenseSet<const SCEV *> &VisitedRegs,
1226 const Loop *L,
Andrew Trick5df90962011-12-06 03:13:31 +00001227 ScalarEvolution &SE, DominatorTree &DT,
Quentin Colombet8aa7abe2013-05-31 17:20:29 +00001228 const LSRUse &LU,
Craig Topper71b7b682014-08-21 05:55:13 +00001229 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
Wei Mi74d5a902017-02-22 21:47:08 +00001230 assert(F.isCanonical(*L) && "Cost is accurate only for canonical formula");
Dan Gohman45774ce2010-02-12 10:34:29 +00001231 // Tally up the registers.
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +00001232 unsigned PrevAddRecCost = AddRecCost;
1233 unsigned PrevNumRegs = NumRegs;
1234 unsigned PrevNumBaseAdds = NumBaseAdds;
Dan Gohman45774ce2010-02-12 10:34:29 +00001235 if (const SCEV *ScaledReg = F.ScaledReg) {
1236 if (VisitedRegs.count(ScaledReg)) {
Tim Northoverbc6659c2014-01-22 13:27:00 +00001237 Lose();
Dan Gohman45774ce2010-02-12 10:34:29 +00001238 return;
1239 }
Andrew Trick5df90962011-12-06 03:13:31 +00001240 RatePrimaryRegister(ScaledReg, Regs, L, SE, DT, LoserRegs);
Andrew Trick784729d2011-09-26 23:11:04 +00001241 if (isLoser())
1242 return;
Dan Gohman45774ce2010-02-12 10:34:29 +00001243 }
Craig Topper042a3922015-05-25 20:01:18 +00001244 for (const SCEV *BaseReg : F.BaseRegs) {
Dan Gohman45774ce2010-02-12 10:34:29 +00001245 if (VisitedRegs.count(BaseReg)) {
Tim Northoverbc6659c2014-01-22 13:27:00 +00001246 Lose();
Dan Gohman45774ce2010-02-12 10:34:29 +00001247 return;
1248 }
Andrew Trick5df90962011-12-06 03:13:31 +00001249 RatePrimaryRegister(BaseReg, Regs, L, SE, DT, LoserRegs);
Andrew Trick784729d2011-09-26 23:11:04 +00001250 if (isLoser())
1251 return;
Dan Gohman45774ce2010-02-12 10:34:29 +00001252 }
1253
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +00001254 // Treat every new register that exceeds TTI.getNumberOfRegisters() - 1 as
1255 // additional instruction (at least fill).
1256 unsigned TTIRegNum = TTI.getNumberOfRegisters(false) - 1;
1257 if (NumRegs > TTIRegNum) {
1258 // Cost already exceeded TTIRegNum, then only newly added register can add
1259 // new instructions.
1260 if (PrevNumRegs > TTIRegNum)
1261 Insns += (NumRegs - PrevNumRegs);
1262 else
1263 Insns += (NumRegs - TTIRegNum);
1264 }
1265
Dan Gohman6136e942011-05-03 00:46:49 +00001266 // Determine how many (unfolded) adds we'll need inside the loop.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001267 size_t NumBaseParts = F.getNumRegs();
Dan Gohman6136e942011-05-03 00:46:49 +00001268 if (NumBaseParts > 1)
Quentin Colombet8aa7abe2013-05-31 17:20:29 +00001269 // Do not count the base and a possible second register if the target
1270 // allows to fold 2 registers.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001271 NumBaseAdds +=
1272 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(TTI, LU, F)));
1273 NumBaseAdds += (F.UnfoldedOffset != 0);
Dan Gohman45774ce2010-02-12 10:34:29 +00001274
Quentin Colombetbf490d42013-05-31 21:29:03 +00001275 // Accumulate non-free scaling amounts.
Wei Mi74d5a902017-02-22 21:47:08 +00001276 ScaleCost += getScalingFactorCost(TTI, LU, F, *L);
Quentin Colombetbf490d42013-05-31 21:29:03 +00001277
Dan Gohman45774ce2010-02-12 10:34:29 +00001278 // Tally up the non-zero immediates.
Jonas Paulsson7a794222016-08-17 13:24:19 +00001279 for (const LSRFixup &Fixup : LU.Fixups) {
1280 int64_t O = Fixup.Offset;
Craig Topper042a3922015-05-25 20:01:18 +00001281 int64_t Offset = (uint64_t)O + F.BaseOffset;
Chandler Carruth6e479322013-01-07 15:04:40 +00001282 if (F.BaseGV)
Dan Gohman45774ce2010-02-12 10:34:29 +00001283 ImmCost += 64; // Handle symbolic values conservatively.
1284 // TODO: This should probably be the pointer size.
1285 else if (Offset != 0)
1286 ImmCost += APInt(64, Offset, true).getMinSignedBits();
Jonas Paulsson7a794222016-08-17 13:24:19 +00001287
1288 // Check with target if this offset with this instruction is
1289 // specifically not supported.
1290 if ((isa<LoadInst>(Fixup.UserInst) || isa<StoreInst>(Fixup.UserInst)) &&
1291 !TTI.isFoldableMemAccessOffset(Fixup.UserInst, Offset))
1292 NumBaseAdds++;
Dan Gohman45774ce2010-02-12 10:34:29 +00001293 }
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +00001294
1295 // If ICmpZero formula ends with not 0, it could not be replaced by
1296 // just add or sub. We'll need to compare final result of AddRec.
1297 // That means we'll need an additional instruction.
1298 // For -10 + {0, +, 1}:
1299 // i = i + 1;
1300 // cmp i, 10
1301 //
1302 // For {-10, +, 1}:
1303 // i = i + 1;
1304 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd())
1305 Insns++;
1306 // Each new AddRec adds 1 instruction to calculation.
1307 Insns += (AddRecCost - PrevAddRecCost);
1308
1309 // BaseAdds adds instructions for unfolded registers.
1310 if (LU.Kind != LSRUse::ICmpZero)
1311 Insns += NumBaseAdds - PrevNumBaseAdds;
Andrew Trick784729d2011-09-26 23:11:04 +00001312 assert(isValid() && "invalid cost");
Dan Gohman45774ce2010-02-12 10:34:29 +00001313}
1314
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001315/// Set this cost to a losing value.
Tim Northoverbc6659c2014-01-22 13:27:00 +00001316void Cost::Lose() {
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +00001317 Insns = ~0u;
Dan Gohman45774ce2010-02-12 10:34:29 +00001318 NumRegs = ~0u;
1319 AddRecCost = ~0u;
1320 NumIVMuls = ~0u;
1321 NumBaseAdds = ~0u;
1322 ImmCost = ~0u;
1323 SetupCost = ~0u;
Quentin Colombetbf490d42013-05-31 21:29:03 +00001324 ScaleCost = ~0u;
Dan Gohman45774ce2010-02-12 10:34:29 +00001325}
1326
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001327/// Choose the lower cost.
Dan Gohman45774ce2010-02-12 10:34:29 +00001328bool Cost::operator<(const Cost &Other) const {
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +00001329 if (InsnsCost && Insns != Other.Insns)
1330 return Insns < Other.Insns;
Benjamin Kramerb2f034b2014-03-03 19:58:30 +00001331 return std::tie(NumRegs, AddRecCost, NumIVMuls, NumBaseAdds, ScaleCost,
1332 ImmCost, SetupCost) <
1333 std::tie(Other.NumRegs, Other.AddRecCost, Other.NumIVMuls,
1334 Other.NumBaseAdds, Other.ScaleCost, Other.ImmCost,
1335 Other.SetupCost);
Dan Gohman45774ce2010-02-12 10:34:29 +00001336}
1337
1338void Cost::print(raw_ostream &OS) const {
Evgeny Stupachenkofe6f5482017-02-11 02:57:43 +00001339 OS << Insns << " instruction" << (Insns == 1 ? " " : "s ");
Dan Gohman45774ce2010-02-12 10:34:29 +00001340 OS << NumRegs << " reg" << (NumRegs == 1 ? "" : "s");
1341 if (AddRecCost != 0)
1342 OS << ", with addrec cost " << AddRecCost;
1343 if (NumIVMuls != 0)
1344 OS << ", plus " << NumIVMuls << " IV mul" << (NumIVMuls == 1 ? "" : "s");
1345 if (NumBaseAdds != 0)
1346 OS << ", plus " << NumBaseAdds << " base add"
1347 << (NumBaseAdds == 1 ? "" : "s");
Quentin Colombetbf490d42013-05-31 21:29:03 +00001348 if (ScaleCost != 0)
1349 OS << ", plus " << ScaleCost << " scale cost";
Dan Gohman45774ce2010-02-12 10:34:29 +00001350 if (ImmCost != 0)
1351 OS << ", plus " << ImmCost << " imm cost";
1352 if (SetupCost != 0)
1353 OS << ", plus " << SetupCost << " setup cost";
1354}
1355
Matthias Braun8c209aa2017-01-28 02:02:38 +00001356#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1357LLVM_DUMP_METHOD void Cost::dump() const {
Dan Gohman45774ce2010-02-12 10:34:29 +00001358 print(errs()); errs() << '\n';
1359}
Matthias Braun8c209aa2017-01-28 02:02:38 +00001360#endif
Dan Gohman45774ce2010-02-12 10:34:29 +00001361
Dan Gohman45774ce2010-02-12 10:34:29 +00001362LSRFixup::LSRFixup()
Jonas Paulsson7a794222016-08-17 13:24:19 +00001363 : UserInst(nullptr), OperandValToReplace(nullptr),
Craig Topperf40110f2014-04-25 05:29:35 +00001364 Offset(0) {}
Dan Gohman45774ce2010-02-12 10:34:29 +00001365
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001366/// Test whether this fixup always uses its value outside of the given loop.
Dan Gohmand006ab92010-04-07 22:27:08 +00001367bool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const {
1368 // PHI nodes use their value in their incoming blocks.
1369 if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) {
1370 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
1371 if (PN->getIncomingValue(i) == OperandValToReplace &&
1372 L->contains(PN->getIncomingBlock(i)))
1373 return false;
1374 return true;
1375 }
1376
1377 return !L->contains(UserInst);
1378}
1379
Dan Gohman45774ce2010-02-12 10:34:29 +00001380void LSRFixup::print(raw_ostream &OS) const {
1381 OS << "UserInst=";
1382 // Store is common and interesting enough to be worth special-casing.
1383 if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
1384 OS << "store ";
Chandler Carruthd48cdbf2014-01-09 02:29:41 +00001385 Store->getOperand(0)->printAsOperand(OS, /*PrintType=*/false);
Dan Gohman45774ce2010-02-12 10:34:29 +00001386 } else if (UserInst->getType()->isVoidTy())
1387 OS << UserInst->getOpcodeName();
1388 else
Chandler Carruthd48cdbf2014-01-09 02:29:41 +00001389 UserInst->printAsOperand(OS, /*PrintType=*/false);
Dan Gohman45774ce2010-02-12 10:34:29 +00001390
1391 OS << ", OperandValToReplace=";
Chandler Carruthd48cdbf2014-01-09 02:29:41 +00001392 OperandValToReplace->printAsOperand(OS, /*PrintType=*/false);
Dan Gohman45774ce2010-02-12 10:34:29 +00001393
Craig Topper042a3922015-05-25 20:01:18 +00001394 for (const Loop *PIL : PostIncLoops) {
Dan Gohman45774ce2010-02-12 10:34:29 +00001395 OS << ", PostIncLoop=";
Craig Topper042a3922015-05-25 20:01:18 +00001396 PIL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
Dan Gohman45774ce2010-02-12 10:34:29 +00001397 }
1398
Dan Gohman45774ce2010-02-12 10:34:29 +00001399 if (Offset != 0)
1400 OS << ", Offset=" << Offset;
1401}
1402
Matthias Braun8c209aa2017-01-28 02:02:38 +00001403#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1404LLVM_DUMP_METHOD void LSRFixup::dump() const {
Dan Gohman45774ce2010-02-12 10:34:29 +00001405 print(errs()); errs() << '\n';
1406}
Matthias Braun8c209aa2017-01-28 02:02:38 +00001407#endif
Dan Gohman45774ce2010-02-12 10:34:29 +00001408
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001409/// Test whether this use as a formula which has the same registers as the given
1410/// formula.
Dan Gohman20fab452010-05-19 23:43:12 +00001411bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const {
Preston Gurd25c3b6a2013-02-01 20:41:27 +00001412 SmallVector<const SCEV *, 4> Key = F.BaseRegs;
Dan Gohman20fab452010-05-19 23:43:12 +00001413 if (F.ScaledReg) Key.push_back(F.ScaledReg);
1414 // Unstable sort by host order ok, because this is only used for uniquifying.
1415 std::sort(Key.begin(), Key.end());
1416 return Uniquifier.count(Key);
1417}
1418
Evgeny Stupachenko9909872e302017-02-21 07:34:40 +00001419/// The function returns a probability of selecting formula without Reg.
1420float LSRUse::getNotSelectedProbability(const SCEV *Reg) const {
1421 unsigned FNum = 0;
1422 for (const Formula &F : Formulae)
1423 if (F.referencesReg(Reg))
1424 FNum++;
1425 return ((float)(Formulae.size() - FNum)) / Formulae.size();
1426}
1427
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001428/// If the given formula has not yet been inserted, add it to the list, and
1429/// return true. Return false otherwise. The formula must be in canonical form.
Wei Mi74d5a902017-02-22 21:47:08 +00001430bool LSRUse::InsertFormula(const Formula &F, const Loop &L) {
1431 assert(F.isCanonical(L) && "Invalid canonical representation");
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001432
Andrew Trick57243da2013-10-25 21:35:56 +00001433 if (!Formulae.empty() && RigidFormula)
1434 return false;
1435
Preston Gurd25c3b6a2013-02-01 20:41:27 +00001436 SmallVector<const SCEV *, 4> Key = F.BaseRegs;
Dan Gohman45774ce2010-02-12 10:34:29 +00001437 if (F.ScaledReg) Key.push_back(F.ScaledReg);
1438 // Unstable sort by host order ok, because this is only used for uniquifying.
1439 std::sort(Key.begin(), Key.end());
1440
1441 if (!Uniquifier.insert(Key).second)
1442 return false;
1443
1444 // Using a register to hold the value of 0 is not profitable.
1445 assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&
1446 "Zero allocated in a scaled register!");
1447#ifndef NDEBUG
Craig Topper042a3922015-05-25 20:01:18 +00001448 for (const SCEV *BaseReg : F.BaseRegs)
1449 assert(!BaseReg->isZero() && "Zero allocated in a base register!");
Dan Gohman45774ce2010-02-12 10:34:29 +00001450#endif
1451
1452 // Add the formula to the list.
1453 Formulae.push_back(F);
1454
1455 // Record registers now being used by this use.
Dan Gohman45774ce2010-02-12 10:34:29 +00001456 Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001457 if (F.ScaledReg)
1458 Regs.insert(F.ScaledReg);
Dan Gohman45774ce2010-02-12 10:34:29 +00001459
1460 return true;
Dan Gohman045f8192010-01-22 00:46:49 +00001461}
1462
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001463/// Remove the given formula from this use's list.
Dan Gohmanf1c7b1b2010-05-18 22:39:15 +00001464void LSRUse::DeleteFormula(Formula &F) {
Dan Gohman80a96082010-05-20 15:17:54 +00001465 if (&F != &Formulae.back())
1466 std::swap(F, Formulae.back());
Dan Gohmanf1c7b1b2010-05-18 22:39:15 +00001467 Formulae.pop_back();
1468}
1469
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001470/// Recompute the Regs field, and update RegUses.
Dan Gohman4cf99b52010-05-18 23:42:37 +00001471void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) {
1472 // Now that we've filtered out some formulae, recompute the Regs set.
Benjamin Kramer1c2beed2015-02-19 17:19:43 +00001473 SmallPtrSet<const SCEV *, 4> OldRegs = std::move(Regs);
Dan Gohman4cf99b52010-05-18 23:42:37 +00001474 Regs.clear();
Benjamin Kramer1c2beed2015-02-19 17:19:43 +00001475 for (const Formula &F : Formulae) {
Dan Gohman4cf99b52010-05-18 23:42:37 +00001476 if (F.ScaledReg) Regs.insert(F.ScaledReg);
1477 Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
1478 }
1479
1480 // Update the RegTracker.
Craig Topper46276792014-08-24 23:23:06 +00001481 for (const SCEV *S : OldRegs)
1482 if (!Regs.count(S))
Sanjoy Das302bfd02015-08-16 18:22:43 +00001483 RegUses.dropRegister(S, LUIdx);
Dan Gohman4cf99b52010-05-18 23:42:37 +00001484}
1485
Dan Gohman45774ce2010-02-12 10:34:29 +00001486void LSRUse::print(raw_ostream &OS) const {
1487 OS << "LSR Use: Kind=";
1488 switch (Kind) {
1489 case Basic: OS << "Basic"; break;
1490 case Special: OS << "Special"; break;
1491 case ICmpZero: OS << "ICmpZero"; break;
1492 case Address:
1493 OS << "Address of ";
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001494 if (AccessTy.MemTy->isPointerTy())
Dan Gohman45774ce2010-02-12 10:34:29 +00001495 OS << "pointer"; // the full pointer type could be really verbose
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001496 else {
1497 OS << *AccessTy.MemTy;
1498 }
1499
1500 OS << " in addrspace(" << AccessTy.AddrSpace << ')';
Evan Cheng133694d2007-10-25 09:11:16 +00001501 }
1502
Dan Gohman45774ce2010-02-12 10:34:29 +00001503 OS << ", Offsets={";
Craig Topper042a3922015-05-25 20:01:18 +00001504 bool NeedComma = false;
Jonas Paulsson7a794222016-08-17 13:24:19 +00001505 for (const LSRFixup &Fixup : Fixups) {
Craig Topper042a3922015-05-25 20:01:18 +00001506 if (NeedComma) OS << ',';
Jonas Paulsson7a794222016-08-17 13:24:19 +00001507 OS << Fixup.Offset;
Craig Topper042a3922015-05-25 20:01:18 +00001508 NeedComma = true;
Dan Gohman045f8192010-01-22 00:46:49 +00001509 }
Dan Gohman45774ce2010-02-12 10:34:29 +00001510 OS << '}';
Dan Gohman045f8192010-01-22 00:46:49 +00001511
Dan Gohman45774ce2010-02-12 10:34:29 +00001512 if (AllFixupsOutsideLoop)
1513 OS << ", all-fixups-outside-loop";
Dan Gohman14152082010-07-15 20:24:58 +00001514
1515 if (WidestFixupType)
1516 OS << ", widest fixup type: " << *WidestFixupType;
Dan Gohman045f8192010-01-22 00:46:49 +00001517}
1518
Matthias Braun8c209aa2017-01-28 02:02:38 +00001519#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1520LLVM_DUMP_METHOD void LSRUse::dump() const {
Dan Gohman45774ce2010-02-12 10:34:29 +00001521 print(errs()); errs() << '\n';
1522}
Matthias Braun8c209aa2017-01-28 02:02:38 +00001523#endif
Dan Gohman045f8192010-01-22 00:46:49 +00001524
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001525static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001526 LSRUse::KindType Kind, MemAccessTy AccessTy,
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001527 GlobalValue *BaseGV, int64_t BaseOffset,
1528 bool HasBaseReg, int64_t Scale) {
Dan Gohman45774ce2010-02-12 10:34:29 +00001529 switch (Kind) {
1530 case LSRUse::Address:
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001531 return TTI.isLegalAddressingMode(AccessTy.MemTy, BaseGV, BaseOffset,
1532 HasBaseReg, Scale, AccessTy.AddrSpace);
Dan Gohman45774ce2010-02-12 10:34:29 +00001533
Dan Gohman45774ce2010-02-12 10:34:29 +00001534 case LSRUse::ICmpZero:
1535 // There's not even a target hook for querying whether it would be legal to
1536 // fold a GV into an ICmp.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001537 if (BaseGV)
Dan Gohman45774ce2010-02-12 10:34:29 +00001538 return false;
1539
1540 // ICmp only has two operands; don't allow more than two non-trivial parts.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001541 if (Scale != 0 && HasBaseReg && BaseOffset != 0)
Dan Gohman45774ce2010-02-12 10:34:29 +00001542 return false;
1543
1544 // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by
1545 // putting the scaled register in the other operand of the icmp.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001546 if (Scale != 0 && Scale != -1)
Dan Gohman45774ce2010-02-12 10:34:29 +00001547 return false;
1548
1549 // If we have low-level target information, ask the target if it can fold an
1550 // integer immediate on an icmp.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001551 if (BaseOffset != 0) {
Jakob Stoklund Olesenf2390e82012-04-05 03:10:56 +00001552 // We have one of:
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001553 // ICmpZero BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset
1554 // ICmpZero -1*ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset
Jakob Stoklund Olesenf2390e82012-04-05 03:10:56 +00001555 // Offs is the ICmp immediate.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001556 if (Scale == 0)
1557 // The cast does the right thing with INT64_MIN.
1558 BaseOffset = -(uint64_t)BaseOffset;
1559 return TTI.isLegalICmpImmediate(BaseOffset);
Dan Gohman045f8192010-01-22 00:46:49 +00001560 }
Dan Gohman45774ce2010-02-12 10:34:29 +00001561
Jakob Stoklund Olesenf2390e82012-04-05 03:10:56 +00001562 // ICmpZero BaseReg + -1*ScaleReg => ICmp BaseReg, ScaleReg
Dan Gohman45774ce2010-02-12 10:34:29 +00001563 return true;
1564
1565 case LSRUse::Basic:
1566 // Only handle single-register values.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001567 return !BaseGV && Scale == 0 && BaseOffset == 0;
Dan Gohman45774ce2010-02-12 10:34:29 +00001568
1569 case LSRUse::Special:
Andrew Trickaca8fb32012-06-15 20:07:26 +00001570 // Special case Basic to handle -1 scales.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001571 return !BaseGV && (Scale == 0 || Scale == -1) && BaseOffset == 0;
Dan Gohman045f8192010-01-22 00:46:49 +00001572 }
1573
David Blaikie46a9f012012-01-20 21:51:11 +00001574 llvm_unreachable("Invalid LSRUse Kind!");
Dan Gohman045f8192010-01-22 00:46:49 +00001575}
1576
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001577static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1578 int64_t MinOffset, int64_t MaxOffset,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001579 LSRUse::KindType Kind, MemAccessTy AccessTy,
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001580 GlobalValue *BaseGV, int64_t BaseOffset,
1581 bool HasBaseReg, int64_t Scale) {
Dan Gohman45774ce2010-02-12 10:34:29 +00001582 // Check for overflow.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001583 if (((int64_t)((uint64_t)BaseOffset + MinOffset) > BaseOffset) !=
Dan Gohman45774ce2010-02-12 10:34:29 +00001584 (MinOffset > 0))
1585 return false;
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001586 MinOffset = (uint64_t)BaseOffset + MinOffset;
1587 if (((int64_t)((uint64_t)BaseOffset + MaxOffset) > BaseOffset) !=
1588 (MaxOffset > 0))
1589 return false;
1590 MaxOffset = (uint64_t)BaseOffset + MaxOffset;
1591
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001592 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MinOffset,
1593 HasBaseReg, Scale) &&
1594 isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MaxOffset,
1595 HasBaseReg, Scale);
1596}
1597
1598static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1599 int64_t MinOffset, int64_t MaxOffset,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001600 LSRUse::KindType Kind, MemAccessTy AccessTy,
Wei Mi74d5a902017-02-22 21:47:08 +00001601 const Formula &F, const Loop &L) {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001602 // For the purpose of isAMCompletelyFolded either having a canonical formula
1603 // or a scale not equal to zero is correct.
1604 // Problems may arise from non canonical formulae having a scale == 0.
1605 // Strictly speaking it would best to just rely on canonical formulae.
1606 // However, when we generate the scaled formulae, we first check that the
1607 // scaling factor is profitable before computing the actual ScaledReg for
1608 // compile time sake.
Wei Mi74d5a902017-02-22 21:47:08 +00001609 assert((F.isCanonical(L) || F.Scale != 0));
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001610 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1611 F.BaseGV, F.BaseOffset, F.HasBaseReg, F.Scale);
1612}
1613
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001614/// Test whether we know how to expand the current formula.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001615static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001616 int64_t MaxOffset, LSRUse::KindType Kind,
1617 MemAccessTy AccessTy, GlobalValue *BaseGV,
1618 int64_t BaseOffset, bool HasBaseReg, int64_t Scale) {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001619 // We know how to expand completely foldable formulae.
1620 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1621 BaseOffset, HasBaseReg, Scale) ||
1622 // Or formulae that use a base register produced by a sum of base
1623 // registers.
1624 (Scale == 1 &&
1625 isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1626 BaseGV, BaseOffset, true, 0));
Dan Gohman045f8192010-01-22 00:46:49 +00001627}
1628
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001629static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001630 int64_t MaxOffset, LSRUse::KindType Kind,
1631 MemAccessTy AccessTy, const Formula &F) {
Chandler Carruth6e479322013-01-07 15:04:40 +00001632 return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, F.BaseGV,
1633 F.BaseOffset, F.HasBaseReg, F.Scale);
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001634}
1635
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001636static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1637 const LSRUse &LU, const Formula &F) {
1638 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1639 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1640 F.Scale);
1641}
Quentin Colombet8aa7abe2013-05-31 17:20:29 +00001642
Quentin Colombetbf490d42013-05-31 21:29:03 +00001643static unsigned getScalingFactorCost(const TargetTransformInfo &TTI,
Wei Mi74d5a902017-02-22 21:47:08 +00001644 const LSRUse &LU, const Formula &F,
1645 const Loop &L) {
Quentin Colombetbf490d42013-05-31 21:29:03 +00001646 if (!F.Scale)
1647 return 0;
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001648
1649 // If the use is not completely folded in that instruction, we will have to
1650 // pay an extra cost only for scale != 1.
1651 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
Wei Mi74d5a902017-02-22 21:47:08 +00001652 LU.AccessTy, F, L))
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001653 return F.Scale != 1;
Quentin Colombetbf490d42013-05-31 21:29:03 +00001654
1655 switch (LU.Kind) {
1656 case LSRUse::Address: {
Quentin Colombet145eb972013-06-19 19:59:41 +00001657 // Check the scaling factor cost with both the min and max offsets.
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001658 int ScaleCostMinOffset = TTI.getScalingFactorCost(
1659 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg,
1660 F.Scale, LU.AccessTy.AddrSpace);
1661 int ScaleCostMaxOffset = TTI.getScalingFactorCost(
1662 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg,
1663 F.Scale, LU.AccessTy.AddrSpace);
Quentin Colombet145eb972013-06-19 19:59:41 +00001664
1665 assert(ScaleCostMinOffset >= 0 && ScaleCostMaxOffset >= 0 &&
1666 "Legal addressing mode has an illegal cost!");
1667 return std::max(ScaleCostMinOffset, ScaleCostMaxOffset);
Quentin Colombetbf490d42013-05-31 21:29:03 +00001668 }
1669 case LSRUse::ICmpZero:
Quentin Colombetbf490d42013-05-31 21:29:03 +00001670 case LSRUse::Basic:
1671 case LSRUse::Special:
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001672 // The use is completely folded, i.e., everything is folded into the
1673 // instruction.
Quentin Colombetbf490d42013-05-31 21:29:03 +00001674 return 0;
1675 }
1676
1677 llvm_unreachable("Invalid LSRUse Kind!");
1678}
1679
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001680static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001681 LSRUse::KindType Kind, MemAccessTy AccessTy,
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001682 GlobalValue *BaseGV, int64_t BaseOffset,
1683 bool HasBaseReg) {
Dan Gohman45774ce2010-02-12 10:34:29 +00001684 // Fast-path: zero is always foldable.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001685 if (BaseOffset == 0 && !BaseGV) return true;
Dan Gohman045f8192010-01-22 00:46:49 +00001686
Dan Gohman45774ce2010-02-12 10:34:29 +00001687 // Conservatively, create an address with an immediate and a
1688 // base and a scale.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001689 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
Dan Gohman045f8192010-01-22 00:46:49 +00001690
Dan Gohman20fab452010-05-19 23:43:12 +00001691 // Canonicalize a scale of 1 to a base register if the formula doesn't
1692 // already have a base register.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001693 if (!HasBaseReg && Scale == 1) {
1694 Scale = 0;
1695 HasBaseReg = true;
Dan Gohman20fab452010-05-19 23:43:12 +00001696 }
1697
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001698 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset,
1699 HasBaseReg, Scale);
Dan Gohman045f8192010-01-22 00:46:49 +00001700}
1701
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001702static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
1703 ScalarEvolution &SE, int64_t MinOffset,
1704 int64_t MaxOffset, LSRUse::KindType Kind,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001705 MemAccessTy AccessTy, const SCEV *S,
1706 bool HasBaseReg) {
Dan Gohman45774ce2010-02-12 10:34:29 +00001707 // Fast-path: zero is always foldable.
1708 if (S->isZero()) return true;
1709
1710 // Conservatively, create an address with an immediate and a
1711 // base and a scale.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001712 int64_t BaseOffset = ExtractImmediate(S, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +00001713 GlobalValue *BaseGV = ExtractSymbol(S, SE);
1714
1715 // If there's anything else involved, it's not foldable.
1716 if (!S->isZero()) return false;
1717
1718 // Fast-path: zero is always foldable.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001719 if (BaseOffset == 0 && !BaseGV) return true;
Dan Gohman45774ce2010-02-12 10:34:29 +00001720
1721 // Conservatively, create an address with an immediate and a
1722 // base and a scale.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001723 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
Dan Gohman45774ce2010-02-12 10:34:29 +00001724
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001725 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1726 BaseOffset, HasBaseReg, Scale);
Dan Gohman045f8192010-01-22 00:46:49 +00001727}
1728
Dan Gohman297fb8b2010-06-19 21:21:39 +00001729namespace {
1730
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001731/// An individual increment in a Chain of IV increments. Relate an IV user to
1732/// an expression that computes the IV it uses from the IV used by the previous
1733/// link in the Chain.
Andrew Trick29fe5f02012-01-09 19:50:34 +00001734///
1735/// For the head of a chain, IncExpr holds the absolute SCEV expression for the
1736/// original IVOperand. The head of the chain's IVOperand is only valid during
1737/// chain collection, before LSR replaces IV users. During chain generation,
1738/// IncExpr can be used to find the new IVOperand that computes the same
1739/// expression.
1740struct IVInc {
1741 Instruction *UserInst;
1742 Value* IVOperand;
1743 const SCEV *IncExpr;
1744
1745 IVInc(Instruction *U, Value *O, const SCEV *E):
1746 UserInst(U), IVOperand(O), IncExpr(E) {}
1747};
1748
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001749// The list of IV increments in program order. We typically add the head of a
1750// chain without finding subsequent links.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001751struct IVChain {
1752 SmallVector<IVInc,1> Incs;
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00001753 const SCEV *ExprBase;
1754
Craig Topperf40110f2014-04-25 05:29:35 +00001755 IVChain() : ExprBase(nullptr) {}
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00001756
1757 IVChain(const IVInc &Head, const SCEV *Base)
1758 : Incs(1, Head), ExprBase(Base) {}
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001759
1760 typedef SmallVectorImpl<IVInc>::const_iterator const_iterator;
1761
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001762 // Return the first increment in the chain.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001763 const_iterator begin() const {
1764 assert(!Incs.empty());
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00001765 return std::next(Incs.begin());
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001766 }
1767 const_iterator end() const {
1768 return Incs.end();
1769 }
1770
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001771 // Returns true if this chain contains any increments.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001772 bool hasIncs() const { return Incs.size() >= 2; }
1773
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001774 // Add an IVInc to the end of this chain.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001775 void add(const IVInc &X) { Incs.push_back(X); }
1776
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001777 // Returns the last UserInst in the chain.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001778 Instruction *tailUserInst() const { return Incs.back().UserInst; }
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00001779
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001780 // Returns true if IncExpr can be profitably added to this chain.
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00001781 bool isProfitableIncrement(const SCEV *OperExpr,
1782 const SCEV *IncExpr,
1783 ScalarEvolution&);
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00001784};
Andrew Trick29fe5f02012-01-09 19:50:34 +00001785
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001786/// Helper for CollectChains to track multiple IV increment uses. Distinguish
1787/// between FarUsers that definitely cross IV increments and NearUsers that may
1788/// be used between IV increments.
Andrew Trick29fe5f02012-01-09 19:50:34 +00001789struct ChainUsers {
1790 SmallPtrSet<Instruction*, 4> FarUsers;
1791 SmallPtrSet<Instruction*, 4> NearUsers;
1792};
1793
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001794/// This class holds state for the main loop strength reduction logic.
Dan Gohman45774ce2010-02-12 10:34:29 +00001795class LSRInstance {
1796 IVUsers &IU;
1797 ScalarEvolution &SE;
1798 DominatorTree &DT;
Dan Gohman607e02b2010-04-09 22:07:05 +00001799 LoopInfo &LI;
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001800 const TargetTransformInfo &TTI;
Dan Gohman45774ce2010-02-12 10:34:29 +00001801 Loop *const L;
1802 bool Changed;
1803
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001804 /// This is the insert position that the current loop's induction variable
1805 /// increment should be placed. In simple loops, this is the latch block's
1806 /// terminator. But in more complicated cases, this is a position which will
1807 /// dominate all the in-loop post-increment users.
Dan Gohman45774ce2010-02-12 10:34:29 +00001808 Instruction *IVIncInsertPos;
1809
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001810 /// Interesting factors between use strides.
Justin Lebar54b0be02016-11-05 16:47:25 +00001811 ///
1812 /// We explicitly use a SetVector which contains a SmallSet, instead of the
1813 /// default, a SmallDenseSet, because we need to use the full range of
1814 /// int64_ts, and there's currently no good way of doing that with
1815 /// SmallDenseSet.
1816 SetVector<int64_t, SmallVector<int64_t, 8>, SmallSet<int64_t, 8>> Factors;
Dan Gohman45774ce2010-02-12 10:34:29 +00001817
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001818 /// Interesting use types, to facilitate truncation reuse.
Chris Lattner229907c2011-07-18 04:54:35 +00001819 SmallSetVector<Type *, 4> Types;
Dan Gohman45774ce2010-02-12 10:34:29 +00001820
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001821 /// The list of interesting uses.
Dan Gohman45774ce2010-02-12 10:34:29 +00001822 SmallVector<LSRUse, 16> Uses;
1823
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001824 /// Track which uses use which register candidates.
Dan Gohman45774ce2010-02-12 10:34:29 +00001825 RegUseTracker RegUses;
1826
Andrew Trick29fe5f02012-01-09 19:50:34 +00001827 // Limit the number of chains to avoid quadratic behavior. We don't expect to
1828 // have more than a few IV increment chains in a loop. Missing a Chain falls
1829 // back to normal LSR behavior for those uses.
1830 static const unsigned MaxChains = 8;
1831
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001832 /// IV users can form a chain of IV increments.
Andrew Trick29fe5f02012-01-09 19:50:34 +00001833 SmallVector<IVChain, MaxChains> IVChainVec;
1834
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001835 /// IV users that belong to profitable IVChains.
Andrew Trick248d4102012-01-09 21:18:52 +00001836 SmallPtrSet<Use*, MaxChains> IVIncSet;
1837
Dan Gohman45774ce2010-02-12 10:34:29 +00001838 void OptimizeShadowIV();
1839 bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse);
1840 ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse);
Dan Gohman4c4043c2010-05-20 20:05:31 +00001841 void OptimizeLoopTermCond();
Dan Gohman45774ce2010-02-12 10:34:29 +00001842
Andrew Trick29fe5f02012-01-09 19:50:34 +00001843 void ChainInstruction(Instruction *UserInst, Instruction *IVOper,
1844 SmallVectorImpl<ChainUsers> &ChainUsersVec);
Andrew Trick248d4102012-01-09 21:18:52 +00001845 void FinalizeChain(IVChain &Chain);
Andrew Trick29fe5f02012-01-09 19:50:34 +00001846 void CollectChains();
Andrew Trick248d4102012-01-09 21:18:52 +00001847 void GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00001848 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
Andrew Trick29fe5f02012-01-09 19:50:34 +00001849
Dan Gohman45774ce2010-02-12 10:34:29 +00001850 void CollectInterestingTypesAndFactors();
1851 void CollectFixupsAndInitialFormulae();
1852
Dan Gohman45774ce2010-02-12 10:34:29 +00001853 // Support for sharing of LSRUses between LSRFixups.
Benjamin Kramer62fb0cf2014-03-15 17:17:48 +00001854 typedef DenseMap<LSRUse::SCEVUseKindPair, size_t> UseMapTy;
Dan Gohman45774ce2010-02-12 10:34:29 +00001855 UseMapTy UseMap;
1856
Dan Gohman110ed642010-09-01 01:45:53 +00001857 bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001858 LSRUse::KindType Kind, MemAccessTy AccessTy);
Dan Gohman45774ce2010-02-12 10:34:29 +00001859
Matt Arsenault427a0fd2015-08-15 00:53:06 +00001860 std::pair<size_t, int64_t> getUse(const SCEV *&Expr, LSRUse::KindType Kind,
1861 MemAccessTy AccessTy);
Dan Gohman45774ce2010-02-12 10:34:29 +00001862
Dan Gohmana7b68d62010-10-07 23:33:43 +00001863 void DeleteUse(LSRUse &LU, size_t LUIdx);
Dan Gohman80a96082010-05-20 15:17:54 +00001864
Dan Gohman110ed642010-09-01 01:45:53 +00001865 LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU);
Dan Gohman20fab452010-05-19 23:43:12 +00001866
Dan Gohman8c16b382010-02-22 04:11:59 +00001867 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
Dan Gohman45774ce2010-02-12 10:34:29 +00001868 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1869 void CountRegisters(const Formula &F, size_t LUIdx);
1870 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
1871
1872 void CollectLoopInvariantFixupsAndFormulae();
1873
1874 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
1875 unsigned Depth = 0);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001876
1877 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
1878 const Formula &Base, unsigned Depth,
1879 size_t Idx, bool IsScaledReg = false);
Dan Gohman45774ce2010-02-12 10:34:29 +00001880 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001881 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1882 const Formula &Base, size_t Idx,
1883 bool IsScaledReg = false);
Dan Gohman45774ce2010-02-12 10:34:29 +00001884 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00001885 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1886 const Formula &Base,
1887 const SmallVectorImpl<int64_t> &Worklist,
1888 size_t Idx, bool IsScaledReg = false);
Dan Gohman45774ce2010-02-12 10:34:29 +00001889 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
1890 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
1891 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
1892 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
1893 void GenerateCrossUseConstantOffsets();
1894 void GenerateAllReuseFormulae();
1895
1896 void FilterOutUndesirableDedicatedRegisters();
Dan Gohmana4eca052010-05-18 22:51:59 +00001897
1898 size_t EstimateSearchSpaceComplexity() const;
Dan Gohmane9e08732010-08-29 16:09:42 +00001899 void NarrowSearchSpaceByDetectingSupersets();
1900 void NarrowSearchSpaceByCollapsingUnrolledCode();
Dan Gohman002ff892010-08-29 16:39:22 +00001901 void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
Evgeny Stupachenko9909872e302017-02-21 07:34:40 +00001902 void NarrowSearchSpaceByDeletingCostlyFormulas();
Dan Gohmane9e08732010-08-29 16:09:42 +00001903 void NarrowSearchSpaceByPickingWinnerRegs();
Dan Gohman45774ce2010-02-12 10:34:29 +00001904 void NarrowSearchSpaceUsingHeuristics();
1905
1906 void SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
1907 Cost &SolutionCost,
1908 SmallVectorImpl<const Formula *> &Workspace,
1909 const Cost &CurCost,
1910 const SmallPtrSet<const SCEV *, 16> &CurRegs,
1911 DenseSet<const SCEV *> &VisitedRegs) const;
1912 void Solve(SmallVectorImpl<const Formula *> &Solution) const;
1913
Dan Gohman607e02b2010-04-09 22:07:05 +00001914 BasicBlock::iterator
1915 HoistInsertPosition(BasicBlock::iterator IP,
1916 const SmallVectorImpl<Instruction *> &Inputs) const;
Andrew Trickc908b432012-01-20 07:41:13 +00001917 BasicBlock::iterator
1918 AdjustInsertPositionForExpand(BasicBlock::iterator IP,
1919 const LSRFixup &LF,
1920 const LSRUse &LU,
1921 SCEVExpander &Rewriter) const;
Dan Gohmand2df6432010-04-09 02:00:38 +00001922
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00001923 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
1924 BasicBlock::iterator IP, SCEVExpander &Rewriter,
1925 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
Jonas Paulsson7a794222016-08-17 13:24:19 +00001926 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00001927 const Formula &F, SCEVExpander &Rewriter,
1928 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
1929 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
Dan Gohman45774ce2010-02-12 10:34:29 +00001930 SCEVExpander &Rewriter,
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00001931 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
Justin Bogner843fb202015-12-15 19:40:57 +00001932 void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution);
Dan Gohman45774ce2010-02-12 10:34:29 +00001933
Andrew Trickdc18e382011-12-13 00:55:33 +00001934public:
Justin Bogner843fb202015-12-15 19:40:57 +00001935 LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT,
1936 LoopInfo &LI, const TargetTransformInfo &TTI);
Dan Gohman45774ce2010-02-12 10:34:29 +00001937
1938 bool getChanged() const { return Changed; }
1939
1940 void print_factors_and_types(raw_ostream &OS) const;
1941 void print_fixups(raw_ostream &OS) const;
1942 void print_uses(raw_ostream &OS) const;
1943 void print(raw_ostream &OS) const;
1944 void dump() const;
1945};
1946
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00001947} // end anonymous namespace
Dan Gohman45774ce2010-02-12 10:34:29 +00001948
Sanjoy Das94c4aec2015-08-16 18:22:46 +00001949/// If IV is used in a int-to-float cast inside the loop then try to eliminate
1950/// the cast operation.
Dan Gohman45774ce2010-02-12 10:34:29 +00001951void LSRInstance::OptimizeShadowIV() {
1952 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
1953 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
1954 return;
1955
1956 for (IVUsers::const_iterator UI = IU.begin(), E = IU.end();
1957 UI != E; /* empty */) {
1958 IVUsers::const_iterator CandidateUI = UI;
1959 ++UI;
1960 Instruction *ShadowUse = CandidateUI->getUser();
Craig Topperf40110f2014-04-25 05:29:35 +00001961 Type *DestTy = nullptr;
Andrew Trick858e9f02011-07-21 01:05:01 +00001962 bool IsSigned = false;
Dan Gohman45774ce2010-02-12 10:34:29 +00001963
1964 /* If shadow use is a int->float cast then insert a second IV
1965 to eliminate this cast.
1966
1967 for (unsigned i = 0; i < n; ++i)
1968 foo((double)i);
1969
1970 is transformed into
1971
1972 double d = 0.0;
1973 for (unsigned i = 0; i < n; ++i, ++d)
1974 foo(d);
1975 */
Andrew Trick858e9f02011-07-21 01:05:01 +00001976 if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) {
1977 IsSigned = false;
Dan Gohman45774ce2010-02-12 10:34:29 +00001978 DestTy = UCast->getDestTy();
Andrew Trick858e9f02011-07-21 01:05:01 +00001979 }
1980 else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) {
1981 IsSigned = true;
Dan Gohman45774ce2010-02-12 10:34:29 +00001982 DestTy = SCast->getDestTy();
Andrew Trick858e9f02011-07-21 01:05:01 +00001983 }
Dan Gohman45774ce2010-02-12 10:34:29 +00001984 if (!DestTy) continue;
1985
Chandler Carruth26c59fa2013-01-07 14:41:08 +00001986 // If target does not support DestTy natively then do not apply
1987 // this transformation.
1988 if (!TTI.isTypeLegal(DestTy)) continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00001989
1990 PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0));
1991 if (!PH) continue;
1992 if (PH->getNumIncomingValues() != 2) continue;
1993
Chris Lattner229907c2011-07-18 04:54:35 +00001994 Type *SrcTy = PH->getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00001995 int Mantissa = DestTy->getFPMantissaWidth();
1996 if (Mantissa == -1) continue;
1997 if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa)
1998 continue;
1999
2000 unsigned Entry, Latch;
2001 if (PH->getIncomingBlock(0) == L->getLoopPreheader()) {
2002 Entry = 0;
2003 Latch = 1;
Dan Gohman045f8192010-01-22 00:46:49 +00002004 } else {
Dan Gohman45774ce2010-02-12 10:34:29 +00002005 Entry = 1;
2006 Latch = 0;
Dan Gohman045f8192010-01-22 00:46:49 +00002007 }
Dan Gohman045f8192010-01-22 00:46:49 +00002008
Dan Gohman45774ce2010-02-12 10:34:29 +00002009 ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry));
2010 if (!Init) continue;
Andrew Trick858e9f02011-07-21 01:05:01 +00002011 Constant *NewInit = ConstantFP::get(DestTy, IsSigned ?
Andrew Trickbd243d02011-07-21 01:45:54 +00002012 (double)Init->getSExtValue() :
2013 (double)Init->getZExtValue());
Dan Gohman045f8192010-01-22 00:46:49 +00002014
Dan Gohman45774ce2010-02-12 10:34:29 +00002015 BinaryOperator *Incr =
2016 dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch));
2017 if (!Incr) continue;
2018 if (Incr->getOpcode() != Instruction::Add
2019 && Incr->getOpcode() != Instruction::Sub)
Dan Gohman045f8192010-01-22 00:46:49 +00002020 continue;
Dan Gohman045f8192010-01-22 00:46:49 +00002021
Dan Gohman45774ce2010-02-12 10:34:29 +00002022 /* Initialize new IV, double d = 0.0 in above example. */
Craig Topperf40110f2014-04-25 05:29:35 +00002023 ConstantInt *C = nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +00002024 if (Incr->getOperand(0) == PH)
2025 C = dyn_cast<ConstantInt>(Incr->getOperand(1));
2026 else if (Incr->getOperand(1) == PH)
2027 C = dyn_cast<ConstantInt>(Incr->getOperand(0));
Dan Gohman045f8192010-01-22 00:46:49 +00002028 else
Dan Gohman045f8192010-01-22 00:46:49 +00002029 continue;
2030
Dan Gohman45774ce2010-02-12 10:34:29 +00002031 if (!C) continue;
Dan Gohman045f8192010-01-22 00:46:49 +00002032
Dan Gohman45774ce2010-02-12 10:34:29 +00002033 // Ignore negative constants, as the code below doesn't handle them
2034 // correctly. TODO: Remove this restriction.
2035 if (!C->getValue().isStrictlyPositive()) continue;
Dan Gohman045f8192010-01-22 00:46:49 +00002036
Dan Gohman45774ce2010-02-12 10:34:29 +00002037 /* Add new PHINode. */
Jay Foad52131342011-03-30 11:28:46 +00002038 PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH);
Dan Gohman045f8192010-01-22 00:46:49 +00002039
Dan Gohman45774ce2010-02-12 10:34:29 +00002040 /* create new increment. '++d' in above example. */
2041 Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue());
2042 BinaryOperator *NewIncr =
2043 BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ?
2044 Instruction::FAdd : Instruction::FSub,
2045 NewPH, CFP, "IV.S.next.", Incr);
Dan Gohman045f8192010-01-22 00:46:49 +00002046
Dan Gohman45774ce2010-02-12 10:34:29 +00002047 NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry));
2048 NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch));
Dan Gohman045f8192010-01-22 00:46:49 +00002049
Dan Gohman45774ce2010-02-12 10:34:29 +00002050 /* Remove cast operation */
2051 ShadowUse->replaceAllUsesWith(NewPH);
2052 ShadowUse->eraseFromParent();
Dan Gohman4c4043c2010-05-20 20:05:31 +00002053 Changed = true;
Dan Gohman45774ce2010-02-12 10:34:29 +00002054 break;
Dan Gohman045f8192010-01-22 00:46:49 +00002055 }
2056}
2057
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002058/// If Cond has an operand that is an expression of an IV, set the IV user and
2059/// stride information and return true, otherwise return false.
Dan Gohmanab5fb7f2010-05-20 19:44:23 +00002060bool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) {
Craig Topper042a3922015-05-25 20:01:18 +00002061 for (IVStrideUse &U : IU)
2062 if (U.getUser() == Cond) {
Dan Gohman45774ce2010-02-12 10:34:29 +00002063 // NOTE: we could handle setcc instructions with multiple uses here, but
2064 // InstCombine does it as well for simple uses, it's not clear that it
2065 // occurs enough in real life to handle.
Craig Topper042a3922015-05-25 20:01:18 +00002066 CondUse = &U;
Dan Gohman45774ce2010-02-12 10:34:29 +00002067 return true;
2068 }
Dan Gohman045f8192010-01-22 00:46:49 +00002069 return false;
Evan Cheng133694d2007-10-25 09:11:16 +00002070}
2071
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002072/// Rewrite the loop's terminating condition if it uses a max computation.
Dan Gohman045f8192010-01-22 00:46:49 +00002073///
2074/// This is a narrow solution to a specific, but acute, problem. For loops
2075/// like this:
2076///
2077/// i = 0;
2078/// do {
2079/// p[i] = 0.0;
2080/// } while (++i < n);
2081///
2082/// the trip count isn't just 'n', because 'n' might not be positive. And
2083/// unfortunately this can come up even for loops where the user didn't use
2084/// a C do-while loop. For example, seemingly well-behaved top-test loops
2085/// will commonly be lowered like this:
2086//
2087/// if (n > 0) {
2088/// i = 0;
2089/// do {
2090/// p[i] = 0.0;
2091/// } while (++i < n);
2092/// }
2093///
2094/// and then it's possible for subsequent optimization to obscure the if
2095/// test in such a way that indvars can't find it.
2096///
2097/// When indvars can't find the if test in loops like this, it creates a
2098/// max expression, which allows it to give the loop a canonical
2099/// induction variable:
2100///
2101/// i = 0;
2102/// max = n < 1 ? 1 : n;
2103/// do {
2104/// p[i] = 0.0;
2105/// } while (++i != max);
2106///
2107/// Canonical induction variables are necessary because the loop passes
2108/// are designed around them. The most obvious example of this is the
2109/// LoopInfo analysis, which doesn't remember trip count values. It
2110/// expects to be able to rediscover the trip count each time it is
Dan Gohman45774ce2010-02-12 10:34:29 +00002111/// needed, and it does this using a simple analysis that only succeeds if
Dan Gohman045f8192010-01-22 00:46:49 +00002112/// the loop has a canonical induction variable.
2113///
2114/// However, when it comes time to generate code, the maximum operation
2115/// can be quite costly, especially if it's inside of an outer loop.
2116///
2117/// This function solves this problem by detecting this type of loop and
2118/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
2119/// the instructions for the maximum computation.
2120///
Dan Gohman45774ce2010-02-12 10:34:29 +00002121ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) {
Dan Gohman045f8192010-01-22 00:46:49 +00002122 // Check that the loop matches the pattern we're looking for.
2123 if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
2124 Cond->getPredicate() != CmpInst::ICMP_NE)
2125 return Cond;
Dan Gohman51ad99d2010-01-21 02:09:26 +00002126
Dan Gohman045f8192010-01-22 00:46:49 +00002127 SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1));
2128 if (!Sel || !Sel->hasOneUse()) return Cond;
Dan Gohman51ad99d2010-01-21 02:09:26 +00002129
Dan Gohman45774ce2010-02-12 10:34:29 +00002130 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
Dan Gohman045f8192010-01-22 00:46:49 +00002131 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
2132 return Cond;
Dan Gohman1d2ded72010-05-03 22:09:21 +00002133 const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1);
Dan Gohman51ad99d2010-01-21 02:09:26 +00002134
Dan Gohman045f8192010-01-22 00:46:49 +00002135 // Add one to the backedge-taken count to get the trip count.
Dan Gohman9b7632d2010-08-16 15:39:27 +00002136 const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount);
Dan Gohman534ba372010-04-24 03:13:44 +00002137 if (IterationCount != SE.getSCEV(Sel)) return Cond;
Dan Gohman045f8192010-01-22 00:46:49 +00002138
Dan Gohman534ba372010-04-24 03:13:44 +00002139 // Check for a max calculation that matches the pattern. There's no check
2140 // for ICMP_ULE here because the comparison would be with zero, which
2141 // isn't interesting.
2142 CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
Craig Topperf40110f2014-04-25 05:29:35 +00002143 const SCEVNAryExpr *Max = nullptr;
Dan Gohman534ba372010-04-24 03:13:44 +00002144 if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) {
2145 Pred = ICmpInst::ICMP_SLE;
2146 Max = S;
2147 } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) {
2148 Pred = ICmpInst::ICMP_SLT;
2149 Max = S;
2150 } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) {
2151 Pred = ICmpInst::ICMP_ULT;
2152 Max = U;
2153 } else {
2154 // No match; bail.
Dan Gohman045f8192010-01-22 00:46:49 +00002155 return Cond;
Dan Gohman534ba372010-04-24 03:13:44 +00002156 }
Dan Gohman045f8192010-01-22 00:46:49 +00002157
2158 // To handle a max with more than two operands, this optimization would
2159 // require additional checking and setup.
2160 if (Max->getNumOperands() != 2)
2161 return Cond;
2162
2163 const SCEV *MaxLHS = Max->getOperand(0);
2164 const SCEV *MaxRHS = Max->getOperand(1);
Dan Gohman534ba372010-04-24 03:13:44 +00002165
2166 // ScalarEvolution canonicalizes constants to the left. For < and >, look
2167 // for a comparison with 1. For <= and >=, a comparison with zero.
2168 if (!MaxLHS ||
2169 (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One)))
2170 return Cond;
2171
Dan Gohman045f8192010-01-22 00:46:49 +00002172 // Check the relevant induction variable for conformance to
2173 // the pattern.
Dan Gohman45774ce2010-02-12 10:34:29 +00002174 const SCEV *IV = SE.getSCEV(Cond->getOperand(0));
Dan Gohman045f8192010-01-22 00:46:49 +00002175 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV);
2176 if (!AR || !AR->isAffine() ||
2177 AR->getStart() != One ||
Dan Gohman45774ce2010-02-12 10:34:29 +00002178 AR->getStepRecurrence(SE) != One)
Dan Gohman045f8192010-01-22 00:46:49 +00002179 return Cond;
2180
2181 assert(AR->getLoop() == L &&
2182 "Loop condition operand is an addrec in a different loop!");
2183
2184 // Check the right operand of the select, and remember it, as it will
2185 // be used in the new comparison instruction.
Craig Topperf40110f2014-04-25 05:29:35 +00002186 Value *NewRHS = nullptr;
Dan Gohman534ba372010-04-24 03:13:44 +00002187 if (ICmpInst::isTrueWhenEqual(Pred)) {
2188 // Look for n+1, and grab n.
2189 if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1)))
Jakub Staszakf6df1e32013-03-24 09:25:47 +00002190 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
2191 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
2192 NewRHS = BO->getOperand(0);
Dan Gohman534ba372010-04-24 03:13:44 +00002193 if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2)))
Jakub Staszakf6df1e32013-03-24 09:25:47 +00002194 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
2195 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
2196 NewRHS = BO->getOperand(0);
Dan Gohman534ba372010-04-24 03:13:44 +00002197 if (!NewRHS)
2198 return Cond;
2199 } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS)
Dan Gohman045f8192010-01-22 00:46:49 +00002200 NewRHS = Sel->getOperand(1);
Dan Gohman45774ce2010-02-12 10:34:29 +00002201 else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS)
Dan Gohman045f8192010-01-22 00:46:49 +00002202 NewRHS = Sel->getOperand(2);
Dan Gohman1081f1a2010-06-22 23:07:13 +00002203 else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS))
2204 NewRHS = SU->getValue();
Dan Gohman534ba372010-04-24 03:13:44 +00002205 else
Dan Gohman1081f1a2010-06-22 23:07:13 +00002206 // Max doesn't match expected pattern.
2207 return Cond;
Dan Gohman045f8192010-01-22 00:46:49 +00002208
2209 // Determine the new comparison opcode. It may be signed or unsigned,
2210 // and the original comparison may be either equality or inequality.
Dan Gohman045f8192010-01-22 00:46:49 +00002211 if (Cond->getPredicate() == CmpInst::ICMP_EQ)
2212 Pred = CmpInst::getInversePredicate(Pred);
2213
2214 // Ok, everything looks ok to change the condition into an SLT or SGE and
2215 // delete the max calculation.
2216 ICmpInst *NewCond =
2217 new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp");
2218
2219 // Delete the max calculation instructions.
2220 Cond->replaceAllUsesWith(NewCond);
2221 CondUse->setUser(NewCond);
2222 Instruction *Cmp = cast<Instruction>(Sel->getOperand(0));
2223 Cond->eraseFromParent();
2224 Sel->eraseFromParent();
2225 if (Cmp->use_empty())
2226 Cmp->eraseFromParent();
2227 return NewCond;
Dan Gohman68e77352008-09-15 21:22:06 +00002228}
2229
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002230/// Change loop terminating condition to use the postinc iv when possible.
Dan Gohman4c4043c2010-05-20 20:05:31 +00002231void
Dan Gohman45774ce2010-02-12 10:34:29 +00002232LSRInstance::OptimizeLoopTermCond() {
2233 SmallPtrSet<Instruction *, 4> PostIncs;
2234
James Molloy196ad082016-08-15 07:53:03 +00002235 // We need a different set of heuristics for rotated and non-rotated loops.
2236 // If a loop is rotated then the latch is also the backedge, so inserting
2237 // post-inc expressions just before the latch is ideal. To reduce live ranges
2238 // it also makes sense to rewrite terminating conditions to use post-inc
2239 // expressions.
2240 //
2241 // If the loop is not rotated then the latch is not a backedge; the latch
2242 // check is done in the loop head. Adding post-inc expressions before the
2243 // latch will cause overlapping live-ranges of pre-inc and post-inc expressions
2244 // in the loop body. In this case we do *not* want to use post-inc expressions
2245 // in the latch check, and we want to insert post-inc expressions before
2246 // the backedge.
Evan Cheng85a9f432009-11-12 07:35:05 +00002247 BasicBlock *LatchBlock = L->getLoopLatch();
Evan Chengba4e5da72009-11-17 18:10:11 +00002248 SmallVector<BasicBlock*, 8> ExitingBlocks;
2249 L->getExitingBlocks(ExitingBlocks);
James Molloy196ad082016-08-15 07:53:03 +00002250 if (llvm::all_of(ExitingBlocks, [&LatchBlock](const BasicBlock *BB) {
2251 return LatchBlock != BB;
2252 })) {
2253 // The backedge doesn't exit the loop; treat this as a head-tested loop.
2254 IVIncInsertPos = LatchBlock->getTerminator();
2255 return;
2256 }
Jim Grosbach60f48542009-11-17 17:53:56 +00002257
James Molloy196ad082016-08-15 07:53:03 +00002258 // Otherwise treat this as a rotated loop.
Craig Topper042a3922015-05-25 20:01:18 +00002259 for (BasicBlock *ExitingBlock : ExitingBlocks) {
Evan Cheng85a9f432009-11-12 07:35:05 +00002260
Dan Gohman45774ce2010-02-12 10:34:29 +00002261 // Get the terminating condition for the loop if possible. If we
Evan Chengba4e5da72009-11-17 18:10:11 +00002262 // can, we want to change it to use a post-incremented version of its
2263 // induction variable, to allow coalescing the live ranges for the IV into
2264 // one register value.
Evan Cheng85a9f432009-11-12 07:35:05 +00002265
Evan Chengba4e5da72009-11-17 18:10:11 +00002266 BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator());
2267 if (!TermBr)
2268 continue;
2269 // FIXME: Overly conservative, termination condition could be an 'or' etc..
2270 if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition()))
2271 continue;
Evan Cheng85a9f432009-11-12 07:35:05 +00002272
Evan Chengba4e5da72009-11-17 18:10:11 +00002273 // Search IVUsesByStride to find Cond's IVUse if there is one.
Craig Topperf40110f2014-04-25 05:29:35 +00002274 IVStrideUse *CondUse = nullptr;
Evan Chengba4e5da72009-11-17 18:10:11 +00002275 ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition());
Dan Gohman45774ce2010-02-12 10:34:29 +00002276 if (!FindIVUserForCond(Cond, CondUse))
Evan Chengba4e5da72009-11-17 18:10:11 +00002277 continue;
2278
Evan Chengba4e5da72009-11-17 18:10:11 +00002279 // If the trip count is computed in terms of a max (due to ScalarEvolution
2280 // being unable to find a sufficient guard, for example), change the loop
2281 // comparison to use SLT or ULT instead of NE.
Dan Gohman45774ce2010-02-12 10:34:29 +00002282 // One consequence of doing this now is that it disrupts the count-down
2283 // optimization. That's not always a bad thing though, because in such
2284 // cases it may still be worthwhile to avoid a max.
2285 Cond = OptimizeMax(Cond, CondUse);
Evan Chengba4e5da72009-11-17 18:10:11 +00002286
Dan Gohman45774ce2010-02-12 10:34:29 +00002287 // If this exiting block dominates the latch block, it may also use
2288 // the post-inc value if it won't be shared with other uses.
2289 // Check for dominance.
2290 if (!DT.dominates(ExitingBlock, LatchBlock))
Dan Gohman045f8192010-01-22 00:46:49 +00002291 continue;
Evan Chengba4e5da72009-11-17 18:10:11 +00002292
Dan Gohman45774ce2010-02-12 10:34:29 +00002293 // Conservatively avoid trying to use the post-inc value in non-latch
2294 // exits if there may be pre-inc users in intervening blocks.
Dan Gohman2d0f96d2010-02-14 03:21:49 +00002295 if (LatchBlock != ExitingBlock)
Dan Gohman45774ce2010-02-12 10:34:29 +00002296 for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI)
2297 // Test if the use is reachable from the exiting block. This dominator
2298 // query is a conservative approximation of reachability.
2299 if (&*UI != CondUse &&
2300 !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) {
2301 // Conservatively assume there may be reuse if the quotient of their
2302 // strides could be a legal scale.
Dan Gohmane637ff52010-04-19 21:48:58 +00002303 const SCEV *A = IU.getStride(*CondUse, L);
2304 const SCEV *B = IU.getStride(*UI, L);
Dan Gohmand006ab92010-04-07 22:27:08 +00002305 if (!A || !B) continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00002306 if (SE.getTypeSizeInBits(A->getType()) !=
2307 SE.getTypeSizeInBits(B->getType())) {
2308 if (SE.getTypeSizeInBits(A->getType()) >
2309 SE.getTypeSizeInBits(B->getType()))
2310 B = SE.getSignExtendExpr(B, A->getType());
2311 else
2312 A = SE.getSignExtendExpr(A, B->getType());
2313 }
2314 if (const SCEVConstant *D =
Dan Gohman4eebb942010-02-19 19:35:48 +00002315 dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) {
Dan Gohman86110fa2010-05-20 22:25:20 +00002316 const ConstantInt *C = D->getValue();
Dan Gohman45774ce2010-02-12 10:34:29 +00002317 // Stride of one or negative one can have reuse with non-addresses.
Dan Gohman86110fa2010-05-20 22:25:20 +00002318 if (C->isOne() || C->isAllOnesValue())
Dan Gohman45774ce2010-02-12 10:34:29 +00002319 goto decline_post_inc;
2320 // Avoid weird situations.
Dan Gohman86110fa2010-05-20 22:25:20 +00002321 if (C->getValue().getMinSignedBits() >= 64 ||
2322 C->getValue().isMinSignedValue())
Dan Gohman45774ce2010-02-12 10:34:29 +00002323 goto decline_post_inc;
2324 // Check for possible scaled-address reuse.
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002325 MemAccessTy AccessTy = getAccessType(UI->getUser());
Chandler Carruth26c59fa2013-01-07 14:41:08 +00002326 int64_t Scale = C->getSExtValue();
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002327 if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr,
2328 /*BaseOffset=*/0,
2329 /*HasBaseReg=*/false, Scale,
2330 AccessTy.AddrSpace))
Dan Gohman45774ce2010-02-12 10:34:29 +00002331 goto decline_post_inc;
Chandler Carruth26c59fa2013-01-07 14:41:08 +00002332 Scale = -Scale;
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002333 if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr,
2334 /*BaseOffset=*/0,
2335 /*HasBaseReg=*/false, Scale,
2336 AccessTy.AddrSpace))
Dan Gohman45774ce2010-02-12 10:34:29 +00002337 goto decline_post_inc;
2338 }
2339 }
2340
David Greene2330f782009-12-23 22:58:38 +00002341 DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "
Dan Gohman45774ce2010-02-12 10:34:29 +00002342 << *Cond << '\n');
Evan Chengba4e5da72009-11-17 18:10:11 +00002343
2344 // It's possible for the setcc instruction to be anywhere in the loop, and
2345 // possible for it to have multiple users. If it is not immediately before
2346 // the exiting block branch, move it.
Dan Gohman45774ce2010-02-12 10:34:29 +00002347 if (&*++BasicBlock::iterator(Cond) != TermBr) {
2348 if (Cond->hasOneUse()) {
Evan Chengba4e5da72009-11-17 18:10:11 +00002349 Cond->moveBefore(TermBr);
2350 } else {
Dan Gohman45774ce2010-02-12 10:34:29 +00002351 // Clone the terminating condition and insert into the loopend.
2352 ICmpInst *OldCond = Cond;
Evan Chengba4e5da72009-11-17 18:10:11 +00002353 Cond = cast<ICmpInst>(Cond->clone());
2354 Cond->setName(L->getHeader()->getName() + ".termcond");
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00002355 ExitingBlock->getInstList().insert(TermBr->getIterator(), Cond);
Evan Chengba4e5da72009-11-17 18:10:11 +00002356
2357 // Clone the IVUse, as the old use still exists!
Andrew Trickfc4ccb22011-06-21 15:43:52 +00002358 CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace());
Dan Gohman45774ce2010-02-12 10:34:29 +00002359 TermBr->replaceUsesOfWith(OldCond, Cond);
Evan Chengba4e5da72009-11-17 18:10:11 +00002360 }
Evan Cheng85a9f432009-11-12 07:35:05 +00002361 }
2362
Evan Chengba4e5da72009-11-17 18:10:11 +00002363 // If we get to here, we know that we can transform the setcc instruction to
2364 // use the post-incremented version of the IV, allowing us to coalesce the
2365 // live ranges for the IV correctly.
Dan Gohmand006ab92010-04-07 22:27:08 +00002366 CondUse->transformToPostInc(L);
Evan Chengba4e5da72009-11-17 18:10:11 +00002367 Changed = true;
2368
Dan Gohman45774ce2010-02-12 10:34:29 +00002369 PostIncs.insert(Cond);
2370 decline_post_inc:;
Dan Gohman51ad99d2010-01-21 02:09:26 +00002371 }
Dan Gohman45774ce2010-02-12 10:34:29 +00002372
2373 // Determine an insertion point for the loop induction variable increment. It
2374 // must dominate all the post-inc comparisons we just set up, and it must
2375 // dominate the loop latch edge.
2376 IVIncInsertPos = L->getLoopLatch()->getTerminator();
Craig Topper46276792014-08-24 23:23:06 +00002377 for (Instruction *Inst : PostIncs) {
Dan Gohman45774ce2010-02-12 10:34:29 +00002378 BasicBlock *BB =
2379 DT.findNearestCommonDominator(IVIncInsertPos->getParent(),
Craig Topper46276792014-08-24 23:23:06 +00002380 Inst->getParent());
2381 if (BB == Inst->getParent())
2382 IVIncInsertPos = Inst;
Dan Gohman45774ce2010-02-12 10:34:29 +00002383 else if (BB != IVIncInsertPos->getParent())
2384 IVIncInsertPos = BB->getTerminator();
2385 }
Dan Gohman51ad99d2010-01-21 02:09:26 +00002386}
2387
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002388/// Determine if the given use can accommodate a fixup at the given offset and
2389/// other details. If so, update the use and return true.
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002390bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset,
2391 bool HasBaseReg, LSRUse::KindType Kind,
2392 MemAccessTy AccessTy) {
Dan Gohman110ed642010-09-01 01:45:53 +00002393 int64_t NewMinOffset = LU.MinOffset;
2394 int64_t NewMaxOffset = LU.MaxOffset;
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002395 MemAccessTy NewAccessTy = AccessTy;
Dan Gohman045f8192010-01-22 00:46:49 +00002396
Dan Gohman45774ce2010-02-12 10:34:29 +00002397 // Check for a mismatched kind. It's tempting to collapse mismatched kinds to
2398 // something conservative, however this can pessimize in the case that one of
2399 // the uses will have all its uses outside the loop, for example.
2400 if (LU.Kind != Kind)
Dan Gohman045f8192010-01-22 00:46:49 +00002401 return false;
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00002402
Dan Gohman45774ce2010-02-12 10:34:29 +00002403 // Check for a mismatched access type, and fall back conservatively as needed.
Dan Gohman32655902010-06-19 21:30:18 +00002404 // TODO: Be less conservative when the type is similar and can use the same
2405 // addressing modes.
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002406 if (Kind == LSRUse::Address) {
Matt Arsenault1f2ca662017-01-30 19:50:17 +00002407 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2408 NewAccessTy = MemAccessTy::getUnknown(AccessTy.MemTy->getContext(),
2409 AccessTy.AddrSpace);
2410 }
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002411 }
Dan Gohman51ad99d2010-01-21 02:09:26 +00002412
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00002413 // Conservatively assume HasBaseReg is true for now.
2414 if (NewOffset < LU.MinOffset) {
2415 if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
2416 LU.MaxOffset - NewOffset, HasBaseReg))
2417 return false;
2418 NewMinOffset = NewOffset;
2419 } else if (NewOffset > LU.MaxOffset) {
2420 if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
2421 NewOffset - LU.MinOffset, HasBaseReg))
2422 return false;
2423 NewMaxOffset = NewOffset;
2424 }
2425
Dan Gohman45774ce2010-02-12 10:34:29 +00002426 // Update the use.
Dan Gohman110ed642010-09-01 01:45:53 +00002427 LU.MinOffset = NewMinOffset;
2428 LU.MaxOffset = NewMaxOffset;
2429 LU.AccessTy = NewAccessTy;
Dan Gohman29916e02010-01-21 22:42:49 +00002430 return true;
2431}
2432
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002433/// Return an LSRUse index and an offset value for a fixup which needs the given
2434/// expression, with the given kind and optional access type. Either reuse an
2435/// existing use or create a new one, as needed.
Matt Arsenault427a0fd2015-08-15 00:53:06 +00002436std::pair<size_t, int64_t> LSRInstance::getUse(const SCEV *&Expr,
2437 LSRUse::KindType Kind,
2438 MemAccessTy AccessTy) {
Dan Gohman45774ce2010-02-12 10:34:29 +00002439 const SCEV *Copy = Expr;
2440 int64_t Offset = ExtractImmediate(Expr, SE);
Evan Cheng85a9f432009-11-12 07:35:05 +00002441
Dan Gohman45774ce2010-02-12 10:34:29 +00002442 // Basic uses can't accept any offset, for example.
Craig Topperf40110f2014-04-25 05:29:35 +00002443 if (!isAlwaysFoldable(TTI, Kind, AccessTy, /*BaseGV=*/ nullptr,
Chandler Carruth26c59fa2013-01-07 14:41:08 +00002444 Offset, /*HasBaseReg=*/ true)) {
Dan Gohman45774ce2010-02-12 10:34:29 +00002445 Expr = Copy;
2446 Offset = 0;
2447 }
2448
2449 std::pair<UseMapTy::iterator, bool> P =
Benjamin Kramer62fb0cf2014-03-15 17:17:48 +00002450 UseMap.insert(std::make_pair(LSRUse::SCEVUseKindPair(Expr, Kind), 0));
Dan Gohman45774ce2010-02-12 10:34:29 +00002451 if (!P.second) {
2452 // A use already existed with this base.
2453 size_t LUIdx = P.first->second;
2454 LSRUse &LU = Uses[LUIdx];
Dan Gohman110ed642010-09-01 01:45:53 +00002455 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
Dan Gohman45774ce2010-02-12 10:34:29 +00002456 // Reuse this use.
2457 return std::make_pair(LUIdx, Offset);
2458 }
2459
2460 // Create a new use.
2461 size_t LUIdx = Uses.size();
2462 P.first->second = LUIdx;
2463 Uses.push_back(LSRUse(Kind, AccessTy));
2464 LSRUse &LU = Uses[LUIdx];
2465
Dan Gohman45774ce2010-02-12 10:34:29 +00002466 LU.MinOffset = Offset;
2467 LU.MaxOffset = Offset;
2468 return std::make_pair(LUIdx, Offset);
2469}
2470
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002471/// Delete the given use from the Uses list.
Dan Gohmana7b68d62010-10-07 23:33:43 +00002472void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
Dan Gohman110ed642010-09-01 01:45:53 +00002473 if (&LU != &Uses.back())
Dan Gohman80a96082010-05-20 15:17:54 +00002474 std::swap(LU, Uses.back());
2475 Uses.pop_back();
Dan Gohmana7b68d62010-10-07 23:33:43 +00002476
2477 // Update RegUses.
Sanjoy Das302bfd02015-08-16 18:22:43 +00002478 RegUses.swapAndDropUse(LUIdx, Uses.size());
Dan Gohman80a96082010-05-20 15:17:54 +00002479}
2480
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002481/// Look for a use distinct from OrigLU which is has a formula that has the same
2482/// registers as the given formula.
Dan Gohman20fab452010-05-19 23:43:12 +00002483LSRUse *
2484LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
Dan Gohman110ed642010-09-01 01:45:53 +00002485 const LSRUse &OrigLU) {
2486 // Search all uses for the formula. This could be more clever.
Dan Gohman20fab452010-05-19 23:43:12 +00002487 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
2488 LSRUse &LU = Uses[LUIdx];
Dan Gohmanb6a520d2010-08-29 15:27:08 +00002489 // Check whether this use is close enough to OrigLU, to see whether it's
2490 // worthwhile looking through its formulae.
2491 // Ignore ICmpZero uses because they may contain formulae generated by
2492 // GenerateICmpZeroScales, in which case adding fixup offsets may
2493 // be invalid.
Dan Gohman20fab452010-05-19 23:43:12 +00002494 if (&LU != &OrigLU &&
2495 LU.Kind != LSRUse::ICmpZero &&
2496 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
Dan Gohman14152082010-07-15 20:24:58 +00002497 LU.WidestFixupType == OrigLU.WidestFixupType &&
Dan Gohman20fab452010-05-19 23:43:12 +00002498 LU.HasFormulaWithSameRegs(OrigF)) {
Dan Gohmanb6a520d2010-08-29 15:27:08 +00002499 // Scan through this use's formulae.
Craig Topper042a3922015-05-25 20:01:18 +00002500 for (const Formula &F : LU.Formulae) {
Dan Gohmanb6a520d2010-08-29 15:27:08 +00002501 // Check to see if this formula has the same registers and symbols
2502 // as OrigF.
Dan Gohman20fab452010-05-19 23:43:12 +00002503 if (F.BaseRegs == OrigF.BaseRegs &&
2504 F.ScaledReg == OrigF.ScaledReg &&
Chandler Carruth6e479322013-01-07 15:04:40 +00002505 F.BaseGV == OrigF.BaseGV &&
2506 F.Scale == OrigF.Scale &&
Dan Gohman6136e942011-05-03 00:46:49 +00002507 F.UnfoldedOffset == OrigF.UnfoldedOffset) {
Chandler Carruth6e479322013-01-07 15:04:40 +00002508 if (F.BaseOffset == 0)
Dan Gohman20fab452010-05-19 23:43:12 +00002509 return &LU;
Dan Gohmanb6a520d2010-08-29 15:27:08 +00002510 // This is the formula where all the registers and symbols matched;
2511 // there aren't going to be any others. Since we declined it, we
Benjamin Kramerbde91762012-06-02 10:20:22 +00002512 // can skip the rest of the formulae and proceed to the next LSRUse.
Dan Gohman20fab452010-05-19 23:43:12 +00002513 break;
2514 }
2515 }
2516 }
2517 }
2518
Dan Gohmanb6a520d2010-08-29 15:27:08 +00002519 // Nothing looked good.
Craig Topperf40110f2014-04-25 05:29:35 +00002520 return nullptr;
Dan Gohman20fab452010-05-19 23:43:12 +00002521}
2522
Dan Gohman45774ce2010-02-12 10:34:29 +00002523void LSRInstance::CollectInterestingTypesAndFactors() {
2524 SmallSetVector<const SCEV *, 4> Strides;
2525
Dan Gohman2446f572010-02-19 00:05:23 +00002526 // Collect interesting types and strides.
Dan Gohmand006ab92010-04-07 22:27:08 +00002527 SmallVector<const SCEV *, 4> Worklist;
Craig Topper042a3922015-05-25 20:01:18 +00002528 for (const IVStrideUse &U : IU) {
2529 const SCEV *Expr = IU.getExpr(U);
Dan Gohman45774ce2010-02-12 10:34:29 +00002530
2531 // Collect interesting types.
Dan Gohmand006ab92010-04-07 22:27:08 +00002532 Types.insert(SE.getEffectiveSCEVType(Expr->getType()));
Dan Gohman45774ce2010-02-12 10:34:29 +00002533
Dan Gohmand006ab92010-04-07 22:27:08 +00002534 // Add strides for mentioned loops.
2535 Worklist.push_back(Expr);
2536 do {
2537 const SCEV *S = Worklist.pop_back_val();
2538 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
Andrew Trickd97b83e2012-03-22 22:42:45 +00002539 if (AR->getLoop() == L)
Andrew Tricke8b4f402011-12-10 00:25:00 +00002540 Strides.insert(AR->getStepRecurrence(SE));
Dan Gohmand006ab92010-04-07 22:27:08 +00002541 Worklist.push_back(AR->getStart());
2542 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
Dan Gohmandd41bba2010-06-21 19:47:52 +00002543 Worklist.append(Add->op_begin(), Add->op_end());
Dan Gohmand006ab92010-04-07 22:27:08 +00002544 }
2545 } while (!Worklist.empty());
Dan Gohman2446f572010-02-19 00:05:23 +00002546 }
2547
2548 // Compute interesting factors from the set of interesting strides.
2549 for (SmallSetVector<const SCEV *, 4>::const_iterator
2550 I = Strides.begin(), E = Strides.end(); I != E; ++I)
Dan Gohman45774ce2010-02-12 10:34:29 +00002551 for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter =
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00002552 std::next(I); NewStrideIter != E; ++NewStrideIter) {
Dan Gohman2446f572010-02-19 00:05:23 +00002553 const SCEV *OldStride = *I;
Dan Gohman45774ce2010-02-12 10:34:29 +00002554 const SCEV *NewStride = *NewStrideIter;
Dan Gohman45774ce2010-02-12 10:34:29 +00002555
2556 if (SE.getTypeSizeInBits(OldStride->getType()) !=
2557 SE.getTypeSizeInBits(NewStride->getType())) {
2558 if (SE.getTypeSizeInBits(OldStride->getType()) >
2559 SE.getTypeSizeInBits(NewStride->getType()))
2560 NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType());
2561 else
2562 OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType());
2563 }
2564 if (const SCEVConstant *Factor =
Dan Gohman4eebb942010-02-19 19:35:48 +00002565 dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride,
2566 SE, true))) {
Sanjoy Das0de2fec2015-12-17 20:28:46 +00002567 if (Factor->getAPInt().getMinSignedBits() <= 64)
2568 Factors.insert(Factor->getAPInt().getSExtValue());
Dan Gohman45774ce2010-02-12 10:34:29 +00002569 } else if (const SCEVConstant *Factor =
Dan Gohman8c16b382010-02-22 04:11:59 +00002570 dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride,
2571 NewStride,
Dan Gohman4eebb942010-02-19 19:35:48 +00002572 SE, true))) {
Sanjoy Das0de2fec2015-12-17 20:28:46 +00002573 if (Factor->getAPInt().getMinSignedBits() <= 64)
2574 Factors.insert(Factor->getAPInt().getSExtValue());
Dan Gohman45774ce2010-02-12 10:34:29 +00002575 }
2576 }
Dan Gohman45774ce2010-02-12 10:34:29 +00002577
2578 // If all uses use the same type, don't bother looking for truncation-based
2579 // reuse.
2580 if (Types.size() == 1)
2581 Types.clear();
2582
2583 DEBUG(print_factors_and_types(dbgs()));
2584}
2585
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002586/// Helper for CollectChains that finds an IV operand (computed by an AddRec in
2587/// this loop) within [OI,OE) or returns OE. If IVUsers mapped Instructions to
2588/// IVStrideUses, we could partially skip this.
Andrew Trick29fe5f02012-01-09 19:50:34 +00002589static User::op_iterator
2590findIVOperand(User::op_iterator OI, User::op_iterator OE,
2591 Loop *L, ScalarEvolution &SE) {
2592 for(; OI != OE; ++OI) {
2593 if (Instruction *Oper = dyn_cast<Instruction>(*OI)) {
2594 if (!SE.isSCEVable(Oper->getType()))
2595 continue;
2596
2597 if (const SCEVAddRecExpr *AR =
2598 dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Oper))) {
2599 if (AR->getLoop() == L)
2600 break;
2601 }
2602 }
2603 }
2604 return OI;
2605}
2606
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002607/// IVChain logic must consistenctly peek base TruncInst operands, so wrap it in
2608/// a convenient helper.
Andrew Trick29fe5f02012-01-09 19:50:34 +00002609static Value *getWideOperand(Value *Oper) {
2610 if (TruncInst *Trunc = dyn_cast<TruncInst>(Oper))
2611 return Trunc->getOperand(0);
2612 return Oper;
2613}
2614
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002615/// Return true if we allow an IV chain to include both types.
Andrew Trick29fe5f02012-01-09 19:50:34 +00002616static bool isCompatibleIVType(Value *LVal, Value *RVal) {
2617 Type *LType = LVal->getType();
2618 Type *RType = RVal->getType();
Mikael Holmenece84cd2017-02-14 06:37:42 +00002619 return (LType == RType) || (LType->isPointerTy() && RType->isPointerTy() &&
2620 // Different address spaces means (possibly)
2621 // different types of the pointer implementation,
2622 // e.g. i16 vs i32 so disallow that.
2623 (LType->getPointerAddressSpace() ==
2624 RType->getPointerAddressSpace()));
Andrew Trick29fe5f02012-01-09 19:50:34 +00002625}
2626
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002627/// Return an approximation of this SCEV expression's "base", or NULL for any
2628/// constant. Returning the expression itself is conservative. Returning a
2629/// deeper subexpression is more precise and valid as long as it isn't less
2630/// complex than another subexpression. For expressions involving multiple
2631/// unscaled values, we need to return the pointer-type SCEVUnknown. This avoids
2632/// forming chains across objects, such as: PrevOper==a[i], IVOper==b[i],
2633/// IVInc==b-a.
Andrew Trickd5d2db92012-01-10 01:45:08 +00002634///
2635/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost
2636/// SCEVUnknown, we simply return the rightmost SCEV operand.
2637static const SCEV *getExprBase(const SCEV *S) {
2638 switch (S->getSCEVType()) {
2639 default: // uncluding scUnknown.
2640 return S;
2641 case scConstant:
Craig Topperf40110f2014-04-25 05:29:35 +00002642 return nullptr;
Andrew Trickd5d2db92012-01-10 01:45:08 +00002643 case scTruncate:
2644 return getExprBase(cast<SCEVTruncateExpr>(S)->getOperand());
2645 case scZeroExtend:
2646 return getExprBase(cast<SCEVZeroExtendExpr>(S)->getOperand());
2647 case scSignExtend:
2648 return getExprBase(cast<SCEVSignExtendExpr>(S)->getOperand());
2649 case scAddExpr: {
2650 // Skip over scaled operands (scMulExpr) to follow add operands as long as
2651 // there's nothing more complex.
2652 // FIXME: not sure if we want to recognize negation.
2653 const SCEVAddExpr *Add = cast<SCEVAddExpr>(S);
2654 for (std::reverse_iterator<SCEVAddExpr::op_iterator> I(Add->op_end()),
2655 E(Add->op_begin()); I != E; ++I) {
2656 const SCEV *SubExpr = *I;
2657 if (SubExpr->getSCEVType() == scAddExpr)
2658 return getExprBase(SubExpr);
2659
2660 if (SubExpr->getSCEVType() != scMulExpr)
2661 return SubExpr;
2662 }
2663 return S; // all operands are scaled, be conservative.
2664 }
2665 case scAddRecExpr:
2666 return getExprBase(cast<SCEVAddRecExpr>(S)->getStart());
2667 }
2668}
2669
Andrew Trick248d4102012-01-09 21:18:52 +00002670/// Return true if the chain increment is profitable to expand into a loop
2671/// invariant value, which may require its own register. A profitable chain
2672/// increment will be an offset relative to the same base. We allow such offsets
2673/// to potentially be used as chain increment as long as it's not obviously
2674/// expensive to expand using real instructions.
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002675bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
2676 const SCEV *IncExpr,
2677 ScalarEvolution &SE) {
2678 // Aggressively form chains when -stress-ivchain.
Andrew Trick248d4102012-01-09 21:18:52 +00002679 if (StressIVChain)
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002680 return true;
Andrew Trick248d4102012-01-09 21:18:52 +00002681
Andrew Trickd5d2db92012-01-10 01:45:08 +00002682 // Do not replace a constant offset from IV head with a nonconstant IV
2683 // increment.
2684 if (!isa<SCEVConstant>(IncExpr)) {
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002685 const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand));
Andrew Trickd5d2db92012-01-10 01:45:08 +00002686 if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr)))
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00002687 return false;
Andrew Trickd5d2db92012-01-10 01:45:08 +00002688 }
2689
2690 SmallPtrSet<const SCEV*, 8> Processed;
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002691 return !isHighCostExpansion(IncExpr, Processed, SE);
Andrew Trick248d4102012-01-09 21:18:52 +00002692}
2693
2694/// Return true if the number of registers needed for the chain is estimated to
2695/// be less than the number required for the individual IV users. First prohibit
2696/// any IV users that keep the IV live across increments (the Users set should
2697/// be empty). Next count the number and type of increments in the chain.
2698///
2699/// Chaining IVs can lead to considerable code bloat if ISEL doesn't
2700/// effectively use postinc addressing modes. Only consider it profitable it the
2701/// increments can be computed in fewer registers when chained.
2702///
2703/// TODO: Consider IVInc free if it's already used in another chains.
2704static bool
Craig Topper71b7b682014-08-21 05:55:13 +00002705isProfitableChain(IVChain &Chain, SmallPtrSetImpl<Instruction*> &Users,
Chandler Carruth26c59fa2013-01-07 14:41:08 +00002706 ScalarEvolution &SE, const TargetTransformInfo &TTI) {
Andrew Trick248d4102012-01-09 21:18:52 +00002707 if (StressIVChain)
2708 return true;
2709
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00002710 if (!Chain.hasIncs())
Andrew Trickd5d2db92012-01-10 01:45:08 +00002711 return false;
2712
2713 if (!Users.empty()) {
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00002714 DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " users:\n";
Craig Topper46276792014-08-24 23:23:06 +00002715 for (Instruction *Inst : Users) {
2716 dbgs() << " " << *Inst << "\n";
Andrew Trickd5d2db92012-01-10 01:45:08 +00002717 });
2718 return false;
2719 }
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00002720 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
Andrew Trickd5d2db92012-01-10 01:45:08 +00002721
2722 // The chain itself may require a register, so intialize cost to 1.
2723 int cost = 1;
2724
2725 // A complete chain likely eliminates the need for keeping the original IV in
2726 // a register. LSR does not currently know how to form a complete chain unless
2727 // the header phi already exists.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00002728 if (isa<PHINode>(Chain.tailUserInst())
2729 && SE.getSCEV(Chain.tailUserInst()) == Chain.Incs[0].IncExpr) {
Andrew Trickd5d2db92012-01-10 01:45:08 +00002730 --cost;
2731 }
Craig Topperf40110f2014-04-25 05:29:35 +00002732 const SCEV *LastIncExpr = nullptr;
Andrew Trickd5d2db92012-01-10 01:45:08 +00002733 unsigned NumConstIncrements = 0;
2734 unsigned NumVarIncrements = 0;
2735 unsigned NumReusedIncrements = 0;
Craig Topper042a3922015-05-25 20:01:18 +00002736 for (const IVInc &Inc : Chain) {
2737 if (Inc.IncExpr->isZero())
Andrew Trickd5d2db92012-01-10 01:45:08 +00002738 continue;
2739
2740 // Incrementing by zero or some constant is neutral. We assume constants can
2741 // be folded into an addressing mode or an add's immediate operand.
Craig Topper042a3922015-05-25 20:01:18 +00002742 if (isa<SCEVConstant>(Inc.IncExpr)) {
Andrew Trickd5d2db92012-01-10 01:45:08 +00002743 ++NumConstIncrements;
2744 continue;
2745 }
2746
Craig Topper042a3922015-05-25 20:01:18 +00002747 if (Inc.IncExpr == LastIncExpr)
Andrew Trickd5d2db92012-01-10 01:45:08 +00002748 ++NumReusedIncrements;
2749 else
2750 ++NumVarIncrements;
2751
Craig Topper042a3922015-05-25 20:01:18 +00002752 LastIncExpr = Inc.IncExpr;
Andrew Trickd5d2db92012-01-10 01:45:08 +00002753 }
2754 // An IV chain with a single increment is handled by LSR's postinc
2755 // uses. However, a chain with multiple increments requires keeping the IV's
2756 // value live longer than it needs to be if chained.
2757 if (NumConstIncrements > 1)
2758 --cost;
2759
2760 // Materializing increment expressions in the preheader that didn't exist in
2761 // the original code may cost a register. For example, sign-extended array
2762 // indices can produce ridiculous increments like this:
2763 // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64)))
2764 cost += NumVarIncrements;
2765
2766 // Reusing variable increments likely saves a register to hold the multiple of
2767 // the stride.
2768 cost -= NumReusedIncrements;
2769
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00002770 DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " Cost: " << cost
2771 << "\n");
Andrew Trickd5d2db92012-01-10 01:45:08 +00002772
2773 return cost < 0;
Andrew Trick248d4102012-01-09 21:18:52 +00002774}
2775
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002776/// Add this IV user to an existing chain or make it the head of a new chain.
Andrew Trick29fe5f02012-01-09 19:50:34 +00002777void LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper,
2778 SmallVectorImpl<ChainUsers> &ChainUsersVec) {
2779 // When IVs are used as types of varying widths, they are generally converted
2780 // to a wider type with some uses remaining narrow under a (free) trunc.
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002781 Value *const NextIV = getWideOperand(IVOper);
2782 const SCEV *const OperExpr = SE.getSCEV(NextIV);
2783 const SCEV *const OperExprBase = getExprBase(OperExpr);
Andrew Trick29fe5f02012-01-09 19:50:34 +00002784
2785 // Visit all existing chains. Check if its IVOper can be computed as a
2786 // profitable loop invariant increment from the last link in the Chain.
2787 unsigned ChainIdx = 0, NChains = IVChainVec.size();
Craig Topperf40110f2014-04-25 05:29:35 +00002788 const SCEV *LastIncExpr = nullptr;
Andrew Trick29fe5f02012-01-09 19:50:34 +00002789 for (; ChainIdx < NChains; ++ChainIdx) {
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002790 IVChain &Chain = IVChainVec[ChainIdx];
2791
2792 // Prune the solution space aggressively by checking that both IV operands
2793 // are expressions that operate on the same unscaled SCEVUnknown. This
2794 // "base" will be canceled by the subsequent getMinusSCEV call. Checking
2795 // first avoids creating extra SCEV expressions.
2796 if (!StressIVChain && Chain.ExprBase != OperExprBase)
2797 continue;
2798
2799 Value *PrevIV = getWideOperand(Chain.Incs.back().IVOperand);
Andrew Trick29fe5f02012-01-09 19:50:34 +00002800 if (!isCompatibleIVType(PrevIV, NextIV))
2801 continue;
2802
Andrew Trick356a8962012-03-26 20:28:35 +00002803 // A phi node terminates a chain.
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002804 if (isa<PHINode>(UserInst) && isa<PHINode>(Chain.tailUserInst()))
Andrew Trick29fe5f02012-01-09 19:50:34 +00002805 continue;
2806
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002807 // The increment must be loop-invariant so it can be kept in a register.
2808 const SCEV *PrevExpr = SE.getSCEV(PrevIV);
2809 const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr);
2810 if (!SE.isLoopInvariant(IncExpr, L))
2811 continue;
2812
2813 if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
Andrew Trick29fe5f02012-01-09 19:50:34 +00002814 LastIncExpr = IncExpr;
2815 break;
2816 }
2817 }
2818 // If we haven't found a chain, create a new one, unless we hit the max. Don't
2819 // bother for phi nodes, because they must be last in the chain.
2820 if (ChainIdx == NChains) {
2821 if (isa<PHINode>(UserInst))
2822 return;
Andrew Trick248d4102012-01-09 21:18:52 +00002823 if (NChains >= MaxChains && !StressIVChain) {
Andrew Trick29fe5f02012-01-09 19:50:34 +00002824 DEBUG(dbgs() << "IV Chain Limit\n");
2825 return;
2826 }
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002827 LastIncExpr = OperExpr;
Andrew Trickb9c822a2012-01-20 21:23:40 +00002828 // IVUsers may have skipped over sign/zero extensions. We don't currently
2829 // attempt to form chains involving extensions unless they can be hoisted
2830 // into this loop's AddRec.
2831 if (!isa<SCEVAddRecExpr>(LastIncExpr))
2832 return;
Andrew Trick29fe5f02012-01-09 19:50:34 +00002833 ++NChains;
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002834 IVChainVec.push_back(IVChain(IVInc(UserInst, IVOper, LastIncExpr),
2835 OperExprBase));
Andrew Trick29fe5f02012-01-09 19:50:34 +00002836 ChainUsersVec.resize(NChains);
Jakob Stoklund Olesen293673d2012-04-25 18:01:32 +00002837 DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst
2838 << ") IV=" << *LastIncExpr << "\n");
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002839 } else {
Jakob Stoklund Olesen293673d2012-04-25 18:01:32 +00002840 DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst
2841 << ") IV+" << *LastIncExpr << "\n");
Jakob Stoklund Olesenc90abc82012-04-26 23:33:11 +00002842 // Add this IV user to the end of the chain.
2843 IVChainVec[ChainIdx].add(IVInc(UserInst, IVOper, LastIncExpr));
2844 }
Andrew Trickbc705902013-02-09 01:11:01 +00002845 IVChain &Chain = IVChainVec[ChainIdx];
Andrew Trick29fe5f02012-01-09 19:50:34 +00002846
2847 SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers;
2848 // This chain's NearUsers become FarUsers.
2849 if (!LastIncExpr->isZero()) {
2850 ChainUsersVec[ChainIdx].FarUsers.insert(NearUsers.begin(),
2851 NearUsers.end());
2852 NearUsers.clear();
2853 }
2854
2855 // All other uses of IVOperand become near uses of the chain.
2856 // We currently ignore intermediate values within SCEV expressions, assuming
2857 // they will eventually be used be the current chain, or can be computed
2858 // from one of the chain increments. To be more precise we could
2859 // transitively follow its user and only add leaf IV users to the set.
Chandler Carruthcdf47882014-03-09 03:16:01 +00002860 for (User *U : IVOper->users()) {
2861 Instruction *OtherUse = dyn_cast<Instruction>(U);
Andrew Trickbc705902013-02-09 01:11:01 +00002862 if (!OtherUse)
Andrew Tricke51feea2012-03-26 18:03:16 +00002863 continue;
Andrew Trickbc705902013-02-09 01:11:01 +00002864 // Uses in the chain will no longer be uses if the chain is formed.
2865 // Include the head of the chain in this iteration (not Chain.begin()).
2866 IVChain::const_iterator IncIter = Chain.Incs.begin();
2867 IVChain::const_iterator IncEnd = Chain.Incs.end();
2868 for( ; IncIter != IncEnd; ++IncIter) {
2869 if (IncIter->UserInst == OtherUse)
2870 break;
2871 }
2872 if (IncIter != IncEnd)
2873 continue;
2874
Andrew Trick29fe5f02012-01-09 19:50:34 +00002875 if (SE.isSCEVable(OtherUse->getType())
2876 && !isa<SCEVUnknown>(SE.getSCEV(OtherUse))
2877 && IU.isIVUserOrOperand(OtherUse)) {
2878 continue;
2879 }
Andrew Tricke51feea2012-03-26 18:03:16 +00002880 NearUsers.insert(OtherUse);
Andrew Trick29fe5f02012-01-09 19:50:34 +00002881 }
2882
2883 // Since this user is part of the chain, it's no longer considered a use
2884 // of the chain.
2885 ChainUsersVec[ChainIdx].FarUsers.erase(UserInst);
2886}
2887
Sanjoy Das94c4aec2015-08-16 18:22:46 +00002888/// Populate the vector of Chains.
Andrew Trick29fe5f02012-01-09 19:50:34 +00002889///
2890/// This decreases ILP at the architecture level. Targets with ample registers,
2891/// multiple memory ports, and no register renaming probably don't want
2892/// this. However, such targets should probably disable LSR altogether.
2893///
2894/// The job of LSR is to make a reasonable choice of induction variables across
2895/// the loop. Subsequent passes can easily "unchain" computation exposing more
2896/// ILP *within the loop* if the target wants it.
2897///
2898/// Finding the best IV chain is potentially a scheduling problem. Since LSR
2899/// will not reorder memory operations, it will recognize this as a chain, but
2900/// will generate redundant IV increments. Ideally this would be corrected later
2901/// by a smart scheduler:
2902/// = A[i]
2903/// = A[i+x]
2904/// A[i] =
2905/// A[i+x] =
2906///
2907/// TODO: Walk the entire domtree within this loop, not just the path to the
2908/// loop latch. This will discover chains on side paths, but requires
2909/// maintaining multiple copies of the Chains state.
2910void LSRInstance::CollectChains() {
Jakob Stoklund Olesen293673d2012-04-25 18:01:32 +00002911 DEBUG(dbgs() << "Collecting IV Chains.\n");
Andrew Trick29fe5f02012-01-09 19:50:34 +00002912 SmallVector<ChainUsers, 8> ChainUsersVec;
2913
2914 SmallVector<BasicBlock *,8> LatchPath;
2915 BasicBlock *LoopHeader = L->getHeader();
2916 for (DomTreeNode *Rung = DT.getNode(L->getLoopLatch());
2917 Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) {
2918 LatchPath.push_back(Rung->getBlock());
2919 }
2920 LatchPath.push_back(LoopHeader);
2921
2922 // Walk the instruction stream from the loop header to the loop latch.
David Majnemerd7708772016-06-24 04:05:21 +00002923 for (BasicBlock *BB : reverse(LatchPath)) {
2924 for (Instruction &I : *BB) {
Andrew Trick29fe5f02012-01-09 19:50:34 +00002925 // Skip instructions that weren't seen by IVUsers analysis.
David Majnemerd7708772016-06-24 04:05:21 +00002926 if (isa<PHINode>(I) || !IU.isIVUserOrOperand(&I))
Andrew Trick29fe5f02012-01-09 19:50:34 +00002927 continue;
2928
2929 // Ignore users that are part of a SCEV expression. This way we only
2930 // consider leaf IV Users. This effectively rediscovers a portion of
2931 // IVUsers analysis but in program order this time.
David Majnemerd7708772016-06-24 04:05:21 +00002932 if (SE.isSCEVable(I.getType()) && !isa<SCEVUnknown>(SE.getSCEV(&I)))
Andrew Trick29fe5f02012-01-09 19:50:34 +00002933 continue;
2934
2935 // Remove this instruction from any NearUsers set it may be in.
2936 for (unsigned ChainIdx = 0, NChains = IVChainVec.size();
2937 ChainIdx < NChains; ++ChainIdx) {
David Majnemerd7708772016-06-24 04:05:21 +00002938 ChainUsersVec[ChainIdx].NearUsers.erase(&I);
Andrew Trick29fe5f02012-01-09 19:50:34 +00002939 }
2940 // Search for operands that can be chained.
2941 SmallPtrSet<Instruction*, 4> UniqueOperands;
David Majnemerd7708772016-06-24 04:05:21 +00002942 User::op_iterator IVOpEnd = I.op_end();
2943 User::op_iterator IVOpIter = findIVOperand(I.op_begin(), IVOpEnd, L, SE);
Andrew Trick29fe5f02012-01-09 19:50:34 +00002944 while (IVOpIter != IVOpEnd) {
2945 Instruction *IVOpInst = cast<Instruction>(*IVOpIter);
David Blaikie70573dc2014-11-19 07:49:26 +00002946 if (UniqueOperands.insert(IVOpInst).second)
David Majnemerd7708772016-06-24 04:05:21 +00002947 ChainInstruction(&I, IVOpInst, ChainUsersVec);
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00002948 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
Andrew Trick29fe5f02012-01-09 19:50:34 +00002949 }
2950 } // Continue walking down the instructions.
2951 } // Continue walking down the domtree.
2952 // Visit phi backedges to determine if the chain can generate the IV postinc.
2953 for (BasicBlock::iterator I = L->getHeader()->begin();
2954 PHINode *PN = dyn_cast<PHINode>(I); ++I) {
2955 if (!SE.isSCEVable(PN->getType()))
2956 continue;
2957
2958 Instruction *IncV =
2959 dyn_cast<Instruction>(PN->getIncomingValueForBlock(L->getLoopLatch()));
2960 if (IncV)
2961 ChainInstruction(PN, IncV, ChainUsersVec);
2962 }
Andrew Trick248d4102012-01-09 21:18:52 +00002963 // Remove any unprofitable chains.
2964 unsigned ChainIdx = 0;
2965 for (unsigned UsersIdx = 0, NChains = IVChainVec.size();
2966 UsersIdx < NChains; ++UsersIdx) {
2967 if (!isProfitableChain(IVChainVec[UsersIdx],
Chandler Carruth26c59fa2013-01-07 14:41:08 +00002968 ChainUsersVec[UsersIdx].FarUsers, SE, TTI))
Andrew Trick248d4102012-01-09 21:18:52 +00002969 continue;
2970 // Preserve the chain at UsesIdx.
2971 if (ChainIdx != UsersIdx)
2972 IVChainVec[ChainIdx] = IVChainVec[UsersIdx];
2973 FinalizeChain(IVChainVec[ChainIdx]);
2974 ++ChainIdx;
2975 }
2976 IVChainVec.resize(ChainIdx);
2977}
2978
2979void LSRInstance::FinalizeChain(IVChain &Chain) {
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00002980 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
2981 DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
Andrew Trick248d4102012-01-09 21:18:52 +00002982
Craig Topper042a3922015-05-25 20:01:18 +00002983 for (const IVInc &Inc : Chain) {
Evgeny Stupachenko8efbe6a2016-11-21 21:55:03 +00002984 DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n");
David Majnemer42531262016-08-12 03:55:06 +00002985 auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand);
Craig Topper042a3922015-05-25 20:01:18 +00002986 assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand");
Andrew Trick248d4102012-01-09 21:18:52 +00002987 IVIncSet.insert(UseI);
2988 }
2989}
2990
2991/// Return true if the IVInc can be folded into an addressing mode.
2992static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst,
Chandler Carruth26c59fa2013-01-07 14:41:08 +00002993 Value *Operand, const TargetTransformInfo &TTI) {
Andrew Trick248d4102012-01-09 21:18:52 +00002994 const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(IncExpr);
2995 if (!IncConst || !isAddressUse(UserInst, Operand))
2996 return false;
2997
Sanjoy Das0de2fec2015-12-17 20:28:46 +00002998 if (IncConst->getAPInt().getMinSignedBits() > 64)
Andrew Trick248d4102012-01-09 21:18:52 +00002999 return false;
3000
Matt Arsenault427a0fd2015-08-15 00:53:06 +00003001 MemAccessTy AccessTy = getAccessType(UserInst);
Andrew Trick248d4102012-01-09 21:18:52 +00003002 int64_t IncOffset = IncConst->getValue()->getSExtValue();
Matt Arsenault427a0fd2015-08-15 00:53:06 +00003003 if (!isAlwaysFoldable(TTI, LSRUse::Address, AccessTy, /*BaseGV=*/nullptr,
3004 IncOffset, /*HaseBaseReg=*/false))
Andrew Trick248d4102012-01-09 21:18:52 +00003005 return false;
3006
3007 return true;
3008}
3009
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003010/// Generate an add or subtract for each IVInc in a chain to materialize the IV
3011/// user's operand from the previous IV user's operand.
Andrew Trick248d4102012-01-09 21:18:52 +00003012void LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00003013 SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
Andrew Trick248d4102012-01-09 21:18:52 +00003014 // Find the new IVOperand for the head of the chain. It may have been replaced
3015 // by LSR.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00003016 const IVInc &Head = Chain.Incs[0];
Andrew Trick248d4102012-01-09 21:18:52 +00003017 User::op_iterator IVOpEnd = Head.UserInst->op_end();
Andrew Trickf3a25442013-03-19 05:10:27 +00003018 // findIVOperand returns IVOpEnd if it can no longer find a valid IV user.
Andrew Trick248d4102012-01-09 21:18:52 +00003019 User::op_iterator IVOpIter = findIVOperand(Head.UserInst->op_begin(),
3020 IVOpEnd, L, SE);
Craig Topperf40110f2014-04-25 05:29:35 +00003021 Value *IVSrc = nullptr;
Andrew Trickf3a25442013-03-19 05:10:27 +00003022 while (IVOpIter != IVOpEnd) {
Andrew Trick248d4102012-01-09 21:18:52 +00003023 IVSrc = getWideOperand(*IVOpIter);
3024
3025 // If this operand computes the expression that the chain needs, we may use
3026 // it. (Check this after setting IVSrc which is used below.)
3027 //
3028 // Note that if Head.IncExpr is wider than IVSrc, then this phi is too
3029 // narrow for the chain, so we can no longer use it. We do allow using a
3030 // wider phi, assuming the LSR checked for free truncation. In that case we
3031 // should already have a truncate on this operand such that
3032 // getSCEV(IVSrc) == IncExpr.
3033 if (SE.getSCEV(*IVOpIter) == Head.IncExpr
3034 || SE.getSCEV(IVSrc) == Head.IncExpr) {
3035 break;
3036 }
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00003037 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
Andrew Trickf3a25442013-03-19 05:10:27 +00003038 }
Andrew Trick248d4102012-01-09 21:18:52 +00003039 if (IVOpIter == IVOpEnd) {
3040 // Gracefully give up on this chain.
3041 DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
3042 return;
3043 }
3044
3045 DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
3046 Type *IVTy = IVSrc->getType();
3047 Type *IntTy = SE.getEffectiveSCEVType(IVTy);
Craig Topperf40110f2014-04-25 05:29:35 +00003048 const SCEV *LeftOverExpr = nullptr;
Craig Topper042a3922015-05-25 20:01:18 +00003049 for (const IVInc &Inc : Chain) {
3050 Instruction *InsertPt = Inc.UserInst;
Andrew Trick248d4102012-01-09 21:18:52 +00003051 if (isa<PHINode>(InsertPt))
3052 InsertPt = L->getLoopLatch()->getTerminator();
3053
3054 // IVOper will replace the current IV User's operand. IVSrc is the IV
3055 // value currently held in a register.
3056 Value *IVOper = IVSrc;
Craig Topper042a3922015-05-25 20:01:18 +00003057 if (!Inc.IncExpr->isZero()) {
Andrew Trick248d4102012-01-09 21:18:52 +00003058 // IncExpr was the result of subtraction of two narrow values, so must
3059 // be signed.
Craig Topper042a3922015-05-25 20:01:18 +00003060 const SCEV *IncExpr = SE.getNoopOrSignExtend(Inc.IncExpr, IntTy);
Andrew Trick248d4102012-01-09 21:18:52 +00003061 LeftOverExpr = LeftOverExpr ?
3062 SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr;
3063 }
3064 if (LeftOverExpr && !LeftOverExpr->isZero()) {
3065 // Expand the IV increment.
3066 Rewriter.clearPostInc();
3067 Value *IncV = Rewriter.expandCodeFor(LeftOverExpr, IntTy, InsertPt);
3068 const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc),
3069 SE.getUnknown(IncV));
3070 IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt);
3071
3072 // If an IV increment can't be folded, use it as the next IV value.
Craig Topper042a3922015-05-25 20:01:18 +00003073 if (!canFoldIVIncExpr(LeftOverExpr, Inc.UserInst, Inc.IVOperand, TTI)) {
Andrew Trick248d4102012-01-09 21:18:52 +00003074 assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
3075 IVSrc = IVOper;
Craig Topperf40110f2014-04-25 05:29:35 +00003076 LeftOverExpr = nullptr;
Andrew Trick248d4102012-01-09 21:18:52 +00003077 }
3078 }
Craig Topper042a3922015-05-25 20:01:18 +00003079 Type *OperTy = Inc.IVOperand->getType();
Andrew Trick248d4102012-01-09 21:18:52 +00003080 if (IVTy != OperTy) {
3081 assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
3082 "cannot extend a chained IV");
3083 IRBuilder<> Builder(InsertPt);
3084 IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain");
3085 }
Craig Topper042a3922015-05-25 20:01:18 +00003086 Inc.UserInst->replaceUsesOfWith(Inc.IVOperand, IVOper);
Benjamin Kramerf5e2fc42015-05-29 19:43:39 +00003087 DeadInsts.emplace_back(Inc.IVOperand);
Andrew Trick248d4102012-01-09 21:18:52 +00003088 }
3089 // If LSR created a new, wider phi, we may also replace its postinc. We only
3090 // do this if we also found a wide value for the head of the chain.
Jakob Stoklund Olesena0337d72012-04-26 23:33:09 +00003091 if (isa<PHINode>(Chain.tailUserInst())) {
Andrew Trick248d4102012-01-09 21:18:52 +00003092 for (BasicBlock::iterator I = L->getHeader()->begin();
3093 PHINode *Phi = dyn_cast<PHINode>(I); ++I) {
3094 if (!isCompatibleIVType(Phi, IVSrc))
3095 continue;
3096 Instruction *PostIncV = dyn_cast<Instruction>(
3097 Phi->getIncomingValueForBlock(L->getLoopLatch()));
3098 if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc)))
3099 continue;
3100 Value *IVOper = IVSrc;
3101 Type *PostIncTy = PostIncV->getType();
3102 if (IVTy != PostIncTy) {
3103 assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types");
3104 IRBuilder<> Builder(L->getLoopLatch()->getTerminator());
3105 Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc());
3106 IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain");
3107 }
3108 Phi->replaceUsesOfWith(PostIncV, IVOper);
Benjamin Kramerf5e2fc42015-05-29 19:43:39 +00003109 DeadInsts.emplace_back(PostIncV);
Andrew Trick248d4102012-01-09 21:18:52 +00003110 }
3111 }
Andrew Trick29fe5f02012-01-09 19:50:34 +00003112}
3113
Dan Gohman45774ce2010-02-12 10:34:29 +00003114void LSRInstance::CollectFixupsAndInitialFormulae() {
Craig Topper042a3922015-05-25 20:01:18 +00003115 for (const IVStrideUse &U : IU) {
3116 Instruction *UserInst = U.getUser();
Andrew Trick248d4102012-01-09 21:18:52 +00003117 // Skip IV users that are part of profitable IV Chains.
David Majnemer42531262016-08-12 03:55:06 +00003118 User::op_iterator UseI =
3119 find(UserInst->operands(), U.getOperandValToReplace());
Andrew Trick248d4102012-01-09 21:18:52 +00003120 assert(UseI != UserInst->op_end() && "cannot find IV operand");
Quentin Colombet35109902017-01-28 01:05:27 +00003121 if (IVIncSet.count(UseI)) {
3122 DEBUG(dbgs() << "Use is in profitable chain: " << **UseI << '\n');
Andrew Trick248d4102012-01-09 21:18:52 +00003123 continue;
Quentin Colombet35109902017-01-28 01:05:27 +00003124 }
Andrew Trick248d4102012-01-09 21:18:52 +00003125
Dan Gohman45774ce2010-02-12 10:34:29 +00003126 LSRUse::KindType Kind = LSRUse::Basic;
Matt Arsenault427a0fd2015-08-15 00:53:06 +00003127 MemAccessTy AccessTy;
Jonas Paulsson7a794222016-08-17 13:24:19 +00003128 if (isAddressUse(UserInst, U.getOperandValToReplace())) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003129 Kind = LSRUse::Address;
Jonas Paulsson7a794222016-08-17 13:24:19 +00003130 AccessTy = getAccessType(UserInst);
Dan Gohman45774ce2010-02-12 10:34:29 +00003131 }
3132
Craig Topper042a3922015-05-25 20:01:18 +00003133 const SCEV *S = IU.getExpr(U);
Jonas Paulsson7a794222016-08-17 13:24:19 +00003134 PostIncLoopSet TmpPostIncLoops = U.getPostIncLoops();
3135
Dan Gohman45774ce2010-02-12 10:34:29 +00003136 // Equality (== and !=) ICmps are special. We can rewrite (i == N) as
3137 // (N - i == 0), and this allows (N - i) to be the expression that we work
3138 // with rather than just N or i, so we can consider the register
3139 // requirements for both N and i at the same time. Limiting this code to
3140 // equality icmps is not a problem because all interesting loops use
3141 // equality icmps, thanks to IndVarSimplify.
Jonas Paulsson7a794222016-08-17 13:24:19 +00003142 if (ICmpInst *CI = dyn_cast<ICmpInst>(UserInst))
Dan Gohman45774ce2010-02-12 10:34:29 +00003143 if (CI->isEquality()) {
3144 // Swap the operands if needed to put the OperandValToReplace on the
3145 // left, for consistency.
3146 Value *NV = CI->getOperand(1);
Jonas Paulsson7a794222016-08-17 13:24:19 +00003147 if (NV == U.getOperandValToReplace()) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003148 CI->setOperand(1, CI->getOperand(0));
3149 CI->setOperand(0, NV);
Dan Gohmanee2fea32010-05-20 19:26:52 +00003150 NV = CI->getOperand(1);
Dan Gohmanfdf98742010-05-20 19:16:03 +00003151 Changed = true;
Dan Gohman45774ce2010-02-12 10:34:29 +00003152 }
3153
3154 // x == y --> x - y == 0
3155 const SCEV *N = SE.getSCEV(NV);
Andrew Trick57243da2013-10-25 21:35:56 +00003156 if (SE.isLoopInvariant(N, L) && isSafeToExpand(N, SE)) {
Dan Gohman3268e4d2011-05-18 21:02:18 +00003157 // S is normalized, so normalize N before folding it into S
3158 // to keep the result normalized.
Sanjoy Dase3a15e82017-04-14 15:49:59 +00003159 N = normalizeForPostIncUse(N, TmpPostIncLoops, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +00003160 Kind = LSRUse::ICmpZero;
3161 S = SE.getMinusSCEV(N, S);
3162 }
3163
3164 // -1 and the negations of all interesting strides (except the negation
3165 // of -1) are now also interesting.
3166 for (size_t i = 0, e = Factors.size(); i != e; ++i)
3167 if (Factors[i] != -1)
3168 Factors.insert(-(uint64_t)Factors[i]);
3169 Factors.insert(-1);
3170 }
3171
Jonas Paulsson7a794222016-08-17 13:24:19 +00003172 // Get or create an LSRUse.
Dan Gohman45774ce2010-02-12 10:34:29 +00003173 std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy);
Jonas Paulsson7a794222016-08-17 13:24:19 +00003174 size_t LUIdx = P.first;
3175 int64_t Offset = P.second;
3176 LSRUse &LU = Uses[LUIdx];
3177
3178 // Record the fixup.
3179 LSRFixup &LF = LU.getNewFixup();
3180 LF.UserInst = UserInst;
3181 LF.OperandValToReplace = U.getOperandValToReplace();
3182 LF.PostIncLoops = TmpPostIncLoops;
3183 LF.Offset = Offset;
Dan Gohmand006ab92010-04-07 22:27:08 +00003184 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
Jonas Paulsson7a794222016-08-17 13:24:19 +00003185
Dan Gohman14152082010-07-15 20:24:58 +00003186 if (!LU.WidestFixupType ||
3187 SE.getTypeSizeInBits(LU.WidestFixupType) <
3188 SE.getTypeSizeInBits(LF.OperandValToReplace->getType()))
3189 LU.WidestFixupType = LF.OperandValToReplace->getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00003190
3191 // If this is the first use of this LSRUse, give it a formula.
3192 if (LU.Formulae.empty()) {
Jonas Paulsson7a794222016-08-17 13:24:19 +00003193 InsertInitialFormula(S, LU, LUIdx);
3194 CountRegisters(LU.Formulae.back(), LUIdx);
Dan Gohman45774ce2010-02-12 10:34:29 +00003195 }
3196 }
3197
3198 DEBUG(print_fixups(dbgs()));
3199}
3200
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003201/// Insert a formula for the given expression into the given use, separating out
3202/// loop-variant portions from loop-invariant and loop-computable portions.
Dan Gohman45774ce2010-02-12 10:34:29 +00003203void
Dan Gohman8c16b382010-02-22 04:11:59 +00003204LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) {
Andrew Trick57243da2013-10-25 21:35:56 +00003205 // Mark uses whose expressions cannot be expanded.
3206 if (!isSafeToExpand(S, SE))
3207 LU.RigidFormula = true;
3208
Dan Gohman45774ce2010-02-12 10:34:29 +00003209 Formula F;
Sanjoy Das302bfd02015-08-16 18:22:43 +00003210 F.initialMatch(S, L, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +00003211 bool Inserted = InsertFormula(LU, LUIdx, F);
3212 assert(Inserted && "Initial formula already exists!"); (void)Inserted;
3213}
3214
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003215/// Insert a simple single-register formula for the given expression into the
3216/// given use.
Dan Gohman45774ce2010-02-12 10:34:29 +00003217void
3218LSRInstance::InsertSupplementalFormula(const SCEV *S,
3219 LSRUse &LU, size_t LUIdx) {
3220 Formula F;
3221 F.BaseRegs.push_back(S);
Chandler Carruth7e31c8f2013-01-12 23:46:04 +00003222 F.HasBaseReg = true;
Dan Gohman45774ce2010-02-12 10:34:29 +00003223 bool Inserted = InsertFormula(LU, LUIdx, F);
3224 assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
3225}
3226
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003227/// Note which registers are used by the given formula, updating RegUses.
Dan Gohman45774ce2010-02-12 10:34:29 +00003228void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
3229 if (F.ScaledReg)
Sanjoy Das302bfd02015-08-16 18:22:43 +00003230 RegUses.countRegister(F.ScaledReg, LUIdx);
Craig Topper042a3922015-05-25 20:01:18 +00003231 for (const SCEV *BaseReg : F.BaseRegs)
Sanjoy Das302bfd02015-08-16 18:22:43 +00003232 RegUses.countRegister(BaseReg, LUIdx);
Dan Gohman45774ce2010-02-12 10:34:29 +00003233}
3234
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003235/// If the given formula has not yet been inserted, add it to the list, and
3236/// return true. Return false otherwise.
Dan Gohman45774ce2010-02-12 10:34:29 +00003237bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003238 // Do not insert formula that we will not be able to expand.
3239 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3240 "Formula is illegal");
Wei Mi74d5a902017-02-22 21:47:08 +00003241
3242 if (!LU.InsertFormula(F, *L))
Dan Gohman45774ce2010-02-12 10:34:29 +00003243 return false;
3244
3245 CountRegisters(F, LUIdx);
3246 return true;
3247}
3248
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003249/// Check for other uses of loop-invariant values which we're tracking. These
3250/// other uses will pin these values in registers, making them less profitable
3251/// for elimination.
Dan Gohman45774ce2010-02-12 10:34:29 +00003252/// TODO: This currently misses non-constant addrec step registers.
3253/// TODO: Should this give more weight to users inside the loop?
3254void
3255LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
3256 SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end());
Andrew Trickdd925ad2014-10-25 19:59:30 +00003257 SmallPtrSet<const SCEV *, 32> Visited;
Dan Gohman45774ce2010-02-12 10:34:29 +00003258
3259 while (!Worklist.empty()) {
3260 const SCEV *S = Worklist.pop_back_val();
3261
Andrew Trick9ccbed52014-10-25 19:42:07 +00003262 // Don't process the same SCEV twice
David Blaikie70573dc2014-11-19 07:49:26 +00003263 if (!Visited.insert(S).second)
Andrew Trick9ccbed52014-10-25 19:42:07 +00003264 continue;
3265
Dan Gohman45774ce2010-02-12 10:34:29 +00003266 if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S))
Dan Gohmandd41bba2010-06-21 19:47:52 +00003267 Worklist.append(N->op_begin(), N->op_end());
Dan Gohman45774ce2010-02-12 10:34:29 +00003268 else if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S))
3269 Worklist.push_back(C->getOperand());
3270 else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
3271 Worklist.push_back(D->getLHS());
3272 Worklist.push_back(D->getRHS());
Chandler Carruthcdf47882014-03-09 03:16:01 +00003273 } else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(S)) {
Chandler Carruthcdf47882014-03-09 03:16:01 +00003274 const Value *V = US->getValue();
Dan Gohman67b44032010-06-04 23:16:05 +00003275 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
3276 // Look for instructions defined outside the loop.
Dan Gohman45774ce2010-02-12 10:34:29 +00003277 if (L->contains(Inst)) continue;
Dan Gohman67b44032010-06-04 23:16:05 +00003278 } else if (isa<UndefValue>(V))
3279 // Undef doesn't have a live range, so it doesn't matter.
3280 continue;
Chandler Carruthcdf47882014-03-09 03:16:01 +00003281 for (const Use &U : V->uses()) {
3282 const Instruction *UserInst = dyn_cast<Instruction>(U.getUser());
Dan Gohman45774ce2010-02-12 10:34:29 +00003283 // Ignore non-instructions.
3284 if (!UserInst)
Dan Gohman045f8192010-01-22 00:46:49 +00003285 continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00003286 // Ignore instructions in other functions (as can happen with
3287 // Constants).
3288 if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
Dan Gohman045f8192010-01-22 00:46:49 +00003289 continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00003290 // Ignore instructions not dominated by the loop.
3291 const BasicBlock *UseBB = !isa<PHINode>(UserInst) ?
3292 UserInst->getParent() :
3293 cast<PHINode>(UserInst)->getIncomingBlock(
Chandler Carruthcdf47882014-03-09 03:16:01 +00003294 PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
Dan Gohman45774ce2010-02-12 10:34:29 +00003295 if (!DT.dominates(L->getHeader(), UseBB))
3296 continue;
David Majnemerb2221842015-11-08 05:04:07 +00003297 // Don't bother if the instruction is in a BB which ends in an EHPad.
3298 if (UseBB->getTerminator()->isEHPad())
3299 continue;
David Majnemerbba17392017-01-13 22:24:27 +00003300 // Don't bother rewriting PHIs in catchswitch blocks.
3301 if (isa<CatchSwitchInst>(UserInst->getParent()->getTerminator()))
3302 continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00003303 // Ignore uses which are part of other SCEV expressions, to avoid
3304 // analyzing them multiple times.
Dan Gohman42ec4eb2010-04-09 19:12:34 +00003305 if (SE.isSCEVable(UserInst->getType())) {
3306 const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst));
3307 // If the user is a no-op, look through to its uses.
3308 if (!isa<SCEVUnknown>(UserS))
3309 continue;
Chandler Carruthcdf47882014-03-09 03:16:01 +00003310 if (UserS == US) {
Dan Gohman42ec4eb2010-04-09 19:12:34 +00003311 Worklist.push_back(
3312 SE.getUnknown(const_cast<Instruction *>(UserInst)));
3313 continue;
3314 }
3315 }
Dan Gohman45774ce2010-02-12 10:34:29 +00003316 // Ignore icmp instructions which are already being analyzed.
3317 if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) {
Chandler Carruthcdf47882014-03-09 03:16:01 +00003318 unsigned OtherIdx = !U.getOperandNo();
Dan Gohman45774ce2010-02-12 10:34:29 +00003319 Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx));
Dan Gohmanafd6db92010-11-17 21:23:15 +00003320 if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L))
Dan Gohman45774ce2010-02-12 10:34:29 +00003321 continue;
3322 }
3323
Matt Arsenault427a0fd2015-08-15 00:53:06 +00003324 std::pair<size_t, int64_t> P = getUse(
3325 S, LSRUse::Basic, MemAccessTy());
Jonas Paulsson7a794222016-08-17 13:24:19 +00003326 size_t LUIdx = P.first;
3327 int64_t Offset = P.second;
3328 LSRUse &LU = Uses[LUIdx];
3329 LSRFixup &LF = LU.getNewFixup();
3330 LF.UserInst = const_cast<Instruction *>(UserInst);
3331 LF.OperandValToReplace = U;
3332 LF.Offset = Offset;
Dan Gohmand006ab92010-04-07 22:27:08 +00003333 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
Dan Gohman14152082010-07-15 20:24:58 +00003334 if (!LU.WidestFixupType ||
3335 SE.getTypeSizeInBits(LU.WidestFixupType) <
3336 SE.getTypeSizeInBits(LF.OperandValToReplace->getType()))
3337 LU.WidestFixupType = LF.OperandValToReplace->getType();
Jonas Paulsson7a794222016-08-17 13:24:19 +00003338 InsertSupplementalFormula(US, LU, LUIdx);
Dan Gohman45774ce2010-02-12 10:34:29 +00003339 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3340 break;
3341 }
3342 }
3343 }
3344}
3345
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003346/// Split S into subexpressions which can be pulled out into separate
3347/// registers. If C is non-null, multiply each subexpression by C.
Andrew Trickc8037062012-07-17 05:30:37 +00003348///
3349/// Return remainder expression after factoring the subexpressions captured by
3350/// Ops. If Ops is complete, return NULL.
3351static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
3352 SmallVectorImpl<const SCEV *> &Ops,
3353 const Loop *L,
3354 ScalarEvolution &SE,
3355 unsigned Depth = 0) {
3356 // Arbitrarily cap recursion to protect compile time.
3357 if (Depth >= 3)
3358 return S;
3359
Dan Gohman45774ce2010-02-12 10:34:29 +00003360 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
3361 // Break out add operands.
Craig Topper042a3922015-05-25 20:01:18 +00003362 for (const SCEV *S : Add->operands()) {
3363 const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1);
Andrew Trickc8037062012-07-17 05:30:37 +00003364 if (Remainder)
3365 Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
3366 }
Craig Topperf40110f2014-04-25 05:29:35 +00003367 return nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +00003368 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
3369 // Split a non-zero base out of an addrec.
Alexandros Lamprineas0ee3ec22016-11-09 08:53:07 +00003370 if (AR->getStart()->isZero() || !AR->isAffine())
Andrew Trickc8037062012-07-17 05:30:37 +00003371 return S;
3372
3373 const SCEV *Remainder = CollectSubexprs(AR->getStart(),
3374 C, Ops, L, SE, Depth+1);
3375 // Split the non-zero AddRec unless it is part of a nested recurrence that
3376 // does not pertain to this loop.
3377 if (Remainder && (AR->getLoop() == L || !isa<SCEVAddRecExpr>(Remainder))) {
3378 Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
Craig Topperf40110f2014-04-25 05:29:35 +00003379 Remainder = nullptr;
Andrew Trickc8037062012-07-17 05:30:37 +00003380 }
3381 if (Remainder != AR->getStart()) {
3382 if (!Remainder)
3383 Remainder = SE.getConstant(AR->getType(), 0);
3384 return SE.getAddRecExpr(Remainder,
3385 AR->getStepRecurrence(SE),
3386 AR->getLoop(),
3387 //FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
3388 SCEV::FlagAnyWrap);
Dan Gohman45774ce2010-02-12 10:34:29 +00003389 }
3390 } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
3391 // Break (C * (a + b + c)) into C*a + C*b + C*c.
Andrew Trickc8037062012-07-17 05:30:37 +00003392 if (Mul->getNumOperands() != 2)
3393 return S;
3394 if (const SCEVConstant *Op0 =
3395 dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
3396 C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
3397 const SCEV *Remainder =
3398 CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1);
3399 if (Remainder)
3400 Ops.push_back(SE.getMulExpr(C, Remainder));
Craig Topperf40110f2014-04-25 05:29:35 +00003401 return nullptr;
Andrew Trickc8037062012-07-17 05:30:37 +00003402 }
Dan Gohman45774ce2010-02-12 10:34:29 +00003403 }
Andrew Trickc8037062012-07-17 05:30:37 +00003404 return S;
Dan Gohman45774ce2010-02-12 10:34:29 +00003405}
3406
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003407/// \brief Helper function for LSRInstance::GenerateReassociations.
3408void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3409 const Formula &Base,
3410 unsigned Depth, size_t Idx,
3411 bool IsScaledReg) {
3412 const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
3413 SmallVector<const SCEV *, 8> AddOps;
3414 const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE);
3415 if (Remainder)
3416 AddOps.push_back(Remainder);
3417
3418 if (AddOps.size() == 1)
3419 return;
3420
3421 for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(),
3422 JE = AddOps.end();
3423 J != JE; ++J) {
3424
3425 // Loop-variant "unknown" values are uninteresting; we won't be able to
3426 // do anything meaningful with them.
3427 if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L))
3428 continue;
3429
3430 // Don't pull a constant into a register if the constant could be folded
3431 // into an immediate field.
3432 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3433 LU.AccessTy, *J, Base.getNumRegs() > 1))
3434 continue;
3435
3436 // Collect all operands except *J.
3437 SmallVector<const SCEV *, 8> InnerAddOps(
3438 ((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J);
3439 InnerAddOps.append(std::next(J),
3440 ((const SmallVector<const SCEV *, 8> &)AddOps).end());
3441
3442 // Don't leave just a constant behind in a register if the constant could
3443 // be folded into an immediate field.
3444 if (InnerAddOps.size() == 1 &&
3445 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3446 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
3447 continue;
3448
3449 const SCEV *InnerSum = SE.getAddExpr(InnerAddOps);
3450 if (InnerSum->isZero())
3451 continue;
3452 Formula F = Base;
3453
3454 // Add the remaining pieces of the add back into the new formula.
3455 const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum);
3456 if (InnerSumSC && SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 &&
3457 TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset +
3458 InnerSumSC->getValue()->getZExtValue())) {
3459 F.UnfoldedOffset =
3460 (uint64_t)F.UnfoldedOffset + InnerSumSC->getValue()->getZExtValue();
3461 if (IsScaledReg)
3462 F.ScaledReg = nullptr;
3463 else
3464 F.BaseRegs.erase(F.BaseRegs.begin() + Idx);
3465 } else if (IsScaledReg)
3466 F.ScaledReg = InnerSum;
3467 else
3468 F.BaseRegs[Idx] = InnerSum;
3469
3470 // Add J as its own register, or an unfolded immediate.
3471 const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J);
3472 if (SC && SE.getTypeSizeInBits(SC->getType()) <= 64 &&
3473 TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset +
3474 SC->getValue()->getZExtValue()))
3475 F.UnfoldedOffset =
3476 (uint64_t)F.UnfoldedOffset + SC->getValue()->getZExtValue();
3477 else
3478 F.BaseRegs.push_back(*J);
3479 // We may have changed the number of register in base regs, adjust the
3480 // formula accordingly.
Wei Mi74d5a902017-02-22 21:47:08 +00003481 F.canonicalize(*L);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003482
3483 if (InsertFormula(LU, LUIdx, F))
3484 // If that formula hadn't been seen before, recurse to find more like
3485 // it.
3486 GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth + 1);
3487 }
3488}
3489
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003490/// Split out subexpressions from adds and the bases of addrecs.
Dan Gohman45774ce2010-02-12 10:34:29 +00003491void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003492 Formula Base, unsigned Depth) {
Wei Mi74d5a902017-02-22 21:47:08 +00003493 assert(Base.isCanonical(*L) && "Input must be in the canonical form");
Dan Gohman45774ce2010-02-12 10:34:29 +00003494 // Arbitrarily cap recursion to protect compile time.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003495 if (Depth >= 3)
3496 return;
Dan Gohman45774ce2010-02-12 10:34:29 +00003497
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003498 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
3499 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
Dan Gohman45774ce2010-02-12 10:34:29 +00003500
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003501 if (Base.Scale == 1)
3502 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
3503 /* Idx */ -1, /* IsScaledReg */ true);
Dan Gohman45774ce2010-02-12 10:34:29 +00003504}
3505
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003506/// Generate a formula consisting of all of the loop-dominating registers added
3507/// into a single register.
Dan Gohman45774ce2010-02-12 10:34:29 +00003508void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
Dan Gohmane4e51a62010-02-14 18:51:39 +00003509 Formula Base) {
Dan Gohman8b0a4192010-03-01 17:49:51 +00003510 // This method is only interesting on a plurality of registers.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003511 if (Base.BaseRegs.size() + (Base.Scale == 1) <= 1)
3512 return;
Dan Gohman45774ce2010-02-12 10:34:29 +00003513
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003514 // Flatten the representation, i.e., reg1 + 1*reg2 => reg1 + reg2, before
3515 // processing the formula.
Sanjoy Das302bfd02015-08-16 18:22:43 +00003516 Base.unscale();
Dan Gohman45774ce2010-02-12 10:34:29 +00003517 Formula F = Base;
3518 F.BaseRegs.clear();
3519 SmallVector<const SCEV *, 4> Ops;
Craig Topper042a3922015-05-25 20:01:18 +00003520 for (const SCEV *BaseReg : Base.BaseRegs) {
Dan Gohman20d9ce22010-11-17 21:41:58 +00003521 if (SE.properlyDominates(BaseReg, L->getHeader()) &&
Dan Gohmanafd6db92010-11-17 21:23:15 +00003522 !SE.hasComputableLoopEvolution(BaseReg, L))
Dan Gohman45774ce2010-02-12 10:34:29 +00003523 Ops.push_back(BaseReg);
3524 else
3525 F.BaseRegs.push_back(BaseReg);
3526 }
3527 if (Ops.size() > 1) {
Dan Gohmanbb7d5222010-02-14 18:50:49 +00003528 const SCEV *Sum = SE.getAddExpr(Ops);
3529 // TODO: If Sum is zero, it probably means ScalarEvolution missed an
3530 // opportunity to fold something. For now, just ignore such cases
Dan Gohman8b0a4192010-03-01 17:49:51 +00003531 // rather than proceed with zero in a register.
Dan Gohmanbb7d5222010-02-14 18:50:49 +00003532 if (!Sum->isZero()) {
3533 F.BaseRegs.push_back(Sum);
Wei Mi74d5a902017-02-22 21:47:08 +00003534 F.canonicalize(*L);
Dan Gohmanbb7d5222010-02-14 18:50:49 +00003535 (void)InsertFormula(LU, LUIdx, F);
3536 }
Dan Gohman45774ce2010-02-12 10:34:29 +00003537 }
3538}
3539
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003540/// \brief Helper function for LSRInstance::GenerateSymbolicOffsets.
3541void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
3542 const Formula &Base, size_t Idx,
3543 bool IsScaledReg) {
3544 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
3545 GlobalValue *GV = ExtractSymbol(G, SE);
3546 if (G->isZero() || !GV)
3547 return;
3548 Formula F = Base;
3549 F.BaseGV = GV;
3550 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3551 return;
3552 if (IsScaledReg)
3553 F.ScaledReg = G;
3554 else
3555 F.BaseRegs[Idx] = G;
3556 (void)InsertFormula(LU, LUIdx, F);
3557}
3558
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003559/// Generate reuse formulae using symbolic offsets.
Dan Gohman45774ce2010-02-12 10:34:29 +00003560void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
3561 Formula Base) {
3562 // We can't add a symbolic offset if the address already contains one.
Chandler Carruth6e479322013-01-07 15:04:40 +00003563 if (Base.BaseGV) return;
Dan Gohman45774ce2010-02-12 10:34:29 +00003564
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003565 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
3566 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
3567 if (Base.Scale == 1)
3568 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
3569 /* IsScaledReg */ true);
3570}
3571
3572/// \brief Helper function for LSRInstance::GenerateConstantOffsets.
3573void LSRInstance::GenerateConstantOffsetsImpl(
3574 LSRUse &LU, unsigned LUIdx, const Formula &Base,
3575 const SmallVectorImpl<int64_t> &Worklist, size_t Idx, bool IsScaledReg) {
3576 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
Craig Topper042a3922015-05-25 20:01:18 +00003577 for (int64_t Offset : Worklist) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003578 Formula F = Base;
Craig Topper042a3922015-05-25 20:01:18 +00003579 F.BaseOffset = (uint64_t)Base.BaseOffset - Offset;
3580 if (isLegalUse(TTI, LU.MinOffset - Offset, LU.MaxOffset - Offset, LU.Kind,
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003581 LU.AccessTy, F)) {
3582 // Add the offset to the base register.
Craig Topper042a3922015-05-25 20:01:18 +00003583 const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), Offset), G);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003584 // If it cancelled out, drop the base register, otherwise update it.
3585 if (NewG->isZero()) {
3586 if (IsScaledReg) {
3587 F.Scale = 0;
3588 F.ScaledReg = nullptr;
3589 } else
Sanjoy Das302bfd02015-08-16 18:22:43 +00003590 F.deleteBaseReg(F.BaseRegs[Idx]);
Wei Mi74d5a902017-02-22 21:47:08 +00003591 F.canonicalize(*L);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003592 } else if (IsScaledReg)
3593 F.ScaledReg = NewG;
3594 else
3595 F.BaseRegs[Idx] = NewG;
3596
3597 (void)InsertFormula(LU, LUIdx, F);
3598 }
Dan Gohman45774ce2010-02-12 10:34:29 +00003599 }
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003600
3601 int64_t Imm = ExtractImmediate(G, SE);
3602 if (G->isZero() || Imm == 0)
3603 return;
3604 Formula F = Base;
3605 F.BaseOffset = (uint64_t)F.BaseOffset + Imm;
3606 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
3607 return;
3608 if (IsScaledReg)
3609 F.ScaledReg = G;
3610 else
3611 F.BaseRegs[Idx] = G;
3612 (void)InsertFormula(LU, LUIdx, F);
Dan Gohman45774ce2010-02-12 10:34:29 +00003613}
3614
3615/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets.
3616void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
3617 Formula Base) {
3618 // TODO: For now, just add the min and max offset, because it usually isn't
3619 // worthwhile looking at everything inbetween.
Dan Gohman4afd4122010-07-15 15:14:45 +00003620 SmallVector<int64_t, 2> Worklist;
Dan Gohman45774ce2010-02-12 10:34:29 +00003621 Worklist.push_back(LU.MinOffset);
3622 if (LU.MaxOffset != LU.MinOffset)
3623 Worklist.push_back(LU.MaxOffset);
3624
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003625 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
3626 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
3627 if (Base.Scale == 1)
3628 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
3629 /* IsScaledReg */ true);
Dan Gohman45774ce2010-02-12 10:34:29 +00003630}
3631
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003632/// For ICmpZero, check to see if we can scale up the comparison. For example, x
3633/// == y -> x*c == y*c.
Dan Gohman45774ce2010-02-12 10:34:29 +00003634void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
3635 Formula Base) {
3636 if (LU.Kind != LSRUse::ICmpZero) return;
3637
3638 // Determine the integer type for the base formula.
Chris Lattner229907c2011-07-18 04:54:35 +00003639 Type *IntTy = Base.getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00003640 if (!IntTy) return;
3641 if (SE.getTypeSizeInBits(IntTy) > 64) return;
3642
3643 // Don't do this if there is more than one offset.
3644 if (LU.MinOffset != LU.MaxOffset) return;
3645
Chandler Carruth6e479322013-01-07 15:04:40 +00003646 assert(!Base.BaseGV && "ICmpZero use is not legal!");
Dan Gohman45774ce2010-02-12 10:34:29 +00003647
3648 // Check each interesting stride.
Craig Topper042a3922015-05-25 20:01:18 +00003649 for (int64_t Factor : Factors) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003650 // Check that the multiplication doesn't overflow.
Chandler Carruth6e479322013-01-07 15:04:40 +00003651 if (Base.BaseOffset == INT64_MIN && Factor == -1)
Dan Gohman5f10d6c2010-02-17 00:41:53 +00003652 continue;
Chandler Carruth6e479322013-01-07 15:04:40 +00003653 int64_t NewBaseOffset = (uint64_t)Base.BaseOffset * Factor;
3654 if (NewBaseOffset / Factor != Base.BaseOffset)
Dan Gohman45774ce2010-02-12 10:34:29 +00003655 continue;
Andrew Trick429e9ed2014-02-26 16:31:56 +00003656 // If the offset will be truncated at this use, check that it is in bounds.
3657 if (!IntTy->isPointerTy() &&
3658 !ConstantInt::isValueValidForType(IntTy, NewBaseOffset))
3659 continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00003660
3661 // Check that multiplying with the use offset doesn't overflow.
3662 int64_t Offset = LU.MinOffset;
Dan Gohman5f10d6c2010-02-17 00:41:53 +00003663 if (Offset == INT64_MIN && Factor == -1)
3664 continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00003665 Offset = (uint64_t)Offset * Factor;
Dan Gohman13ac3b22010-02-17 00:42:19 +00003666 if (Offset / Factor != LU.MinOffset)
Dan Gohman45774ce2010-02-12 10:34:29 +00003667 continue;
Andrew Trick429e9ed2014-02-26 16:31:56 +00003668 // If the offset will be truncated at this use, check that it is in bounds.
3669 if (!IntTy->isPointerTy() &&
3670 !ConstantInt::isValueValidForType(IntTy, Offset))
3671 continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00003672
Dan Gohman963b1c12010-06-24 16:57:52 +00003673 Formula F = Base;
Chandler Carruth6e479322013-01-07 15:04:40 +00003674 F.BaseOffset = NewBaseOffset;
Dan Gohman963b1c12010-06-24 16:57:52 +00003675
Dan Gohman45774ce2010-02-12 10:34:29 +00003676 // Check that this scale is legal.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00003677 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
Dan Gohman45774ce2010-02-12 10:34:29 +00003678 continue;
3679
3680 // Compensate for the use having MinOffset built into it.
Chandler Carruth6e479322013-01-07 15:04:40 +00003681 F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset;
Dan Gohman45774ce2010-02-12 10:34:29 +00003682
Dan Gohman1d2ded72010-05-03 22:09:21 +00003683 const SCEV *FactorS = SE.getConstant(IntTy, Factor);
Dan Gohman45774ce2010-02-12 10:34:29 +00003684
3685 // Check that multiplying with each base register doesn't overflow.
3686 for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) {
3687 F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS);
Dan Gohman4eebb942010-02-19 19:35:48 +00003688 if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i])
Dan Gohman45774ce2010-02-12 10:34:29 +00003689 goto next;
3690 }
3691
3692 // Check that multiplying with the scaled register doesn't overflow.
3693 if (F.ScaledReg) {
3694 F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS);
Dan Gohman4eebb942010-02-19 19:35:48 +00003695 if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg)
Dan Gohman45774ce2010-02-12 10:34:29 +00003696 continue;
3697 }
3698
Dan Gohman6136e942011-05-03 00:46:49 +00003699 // Check that multiplying with the unfolded offset doesn't overflow.
3700 if (F.UnfoldedOffset != 0) {
Dan Gohman6c4a3192011-05-23 21:07:39 +00003701 if (F.UnfoldedOffset == INT64_MIN && Factor == -1)
3702 continue;
Dan Gohman6136e942011-05-03 00:46:49 +00003703 F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset * Factor;
3704 if (F.UnfoldedOffset / Factor != Base.UnfoldedOffset)
3705 continue;
Andrew Trick429e9ed2014-02-26 16:31:56 +00003706 // If the offset will be truncated, check that it is in bounds.
3707 if (!IntTy->isPointerTy() &&
3708 !ConstantInt::isValueValidForType(IntTy, F.UnfoldedOffset))
3709 continue;
Dan Gohman6136e942011-05-03 00:46:49 +00003710 }
3711
Dan Gohman45774ce2010-02-12 10:34:29 +00003712 // If we make it here and it's legal, add it.
3713 (void)InsertFormula(LU, LUIdx, F);
3714 next:;
3715 }
3716}
3717
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003718/// Generate stride factor reuse formulae by making use of scaled-offset address
3719/// modes, for example.
Dan Gohmanab5fb7f2010-05-20 19:44:23 +00003720void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003721 // Determine the integer type for the base formula.
Chris Lattner229907c2011-07-18 04:54:35 +00003722 Type *IntTy = Base.getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00003723 if (!IntTy) return;
3724
3725 // If this Formula already has a scaled register, we can't add another one.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003726 // Try to unscale the formula to generate a better scale.
Sanjoy Das302bfd02015-08-16 18:22:43 +00003727 if (Base.Scale != 0 && !Base.unscale())
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003728 return;
3729
Sanjoy Das302bfd02015-08-16 18:22:43 +00003730 assert(Base.Scale == 0 && "unscale did not did its job!");
Dan Gohman45774ce2010-02-12 10:34:29 +00003731
3732 // Check each interesting stride.
Craig Topper042a3922015-05-25 20:01:18 +00003733 for (int64_t Factor : Factors) {
Chandler Carruth6e479322013-01-07 15:04:40 +00003734 Base.Scale = Factor;
3735 Base.HasBaseReg = Base.BaseRegs.size() > 1;
Dan Gohman45774ce2010-02-12 10:34:29 +00003736 // Check whether this scale is going to be legal.
Chandler Carruth26c59fa2013-01-07 14:41:08 +00003737 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
3738 Base)) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003739 // As a special-case, handle special out-of-loop Basic users specially.
3740 // TODO: Reconsider this special case.
3741 if (LU.Kind == LSRUse::Basic &&
Chandler Carruth26c59fa2013-01-07 14:41:08 +00003742 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
3743 LU.AccessTy, Base) &&
Dan Gohman45774ce2010-02-12 10:34:29 +00003744 LU.AllFixupsOutsideLoop)
3745 LU.Kind = LSRUse::Special;
3746 else
3747 continue;
3748 }
3749 // For an ICmpZero, negating a solitary base register won't lead to
3750 // new solutions.
3751 if (LU.Kind == LSRUse::ICmpZero &&
Chandler Carruth6e479322013-01-07 15:04:40 +00003752 !Base.HasBaseReg && Base.BaseOffset == 0 && !Base.BaseGV)
Dan Gohman45774ce2010-02-12 10:34:29 +00003753 continue;
Wei Mi74d5a902017-02-22 21:47:08 +00003754 // For each addrec base reg, if its loop is current loop, apply the scale.
3755 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) {
3756 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i]);
3757 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
Dan Gohman1d2ded72010-05-03 22:09:21 +00003758 const SCEV *FactorS = SE.getConstant(IntTy, Factor);
Dan Gohman45774ce2010-02-12 10:34:29 +00003759 if (FactorS->isZero())
3760 continue;
3761 // Divide out the factor, ignoring high bits, since we'll be
3762 // scaling the value back up in the end.
Dan Gohman4eebb942010-02-19 19:35:48 +00003763 if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003764 // TODO: This could be optimized to avoid all the copying.
3765 Formula F = Base;
3766 F.ScaledReg = Quotient;
Sanjoy Das302bfd02015-08-16 18:22:43 +00003767 F.deleteBaseReg(F.BaseRegs[i]);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003768 // The canonical representation of 1*reg is reg, which is already in
3769 // Base. In that case, do not try to insert the formula, it will be
3770 // rejected anyway.
Wei Mi74d5a902017-02-22 21:47:08 +00003771 if (F.Scale == 1 && (F.BaseRegs.empty() ||
3772 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003773 continue;
Wei Mi74d5a902017-02-22 21:47:08 +00003774 // If AllFixupsOutsideLoop is true and F.Scale is 1, we may generate
3775 // non canonical Formula with ScaledReg's loop not being L.
3776 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
3777 F.canonicalize(*L);
Dan Gohman45774ce2010-02-12 10:34:29 +00003778 (void)InsertFormula(LU, LUIdx, F);
3779 }
3780 }
Wei Mi74d5a902017-02-22 21:47:08 +00003781 }
Dan Gohman45774ce2010-02-12 10:34:29 +00003782 }
3783}
3784
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003785/// Generate reuse formulae from different IV types.
Dan Gohmanab5fb7f2010-05-20 19:44:23 +00003786void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003787 // Don't bother truncating symbolic values.
Chandler Carruth6e479322013-01-07 15:04:40 +00003788 if (Base.BaseGV) return;
Dan Gohman45774ce2010-02-12 10:34:29 +00003789
3790 // Determine the integer type for the base formula.
Chris Lattner229907c2011-07-18 04:54:35 +00003791 Type *DstTy = Base.getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00003792 if (!DstTy) return;
3793 DstTy = SE.getEffectiveSCEVType(DstTy);
3794
Craig Topper042a3922015-05-25 20:01:18 +00003795 for (Type *SrcTy : Types) {
Chandler Carruth26c59fa2013-01-07 14:41:08 +00003796 if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003797 Formula F = Base;
3798
Craig Topper042a3922015-05-25 20:01:18 +00003799 if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, SrcTy);
3800 for (const SCEV *&BaseReg : F.BaseRegs)
3801 BaseReg = SE.getAnyExtendExpr(BaseReg, SrcTy);
Dan Gohman45774ce2010-02-12 10:34:29 +00003802
3803 // TODO: This assumes we've done basic processing on all uses and
3804 // have an idea what the register usage is.
3805 if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses))
3806 continue;
3807
3808 (void)InsertFormula(LU, LUIdx, F);
3809 }
3810 }
3811}
3812
3813namespace {
3814
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003815/// Helper class for GenerateCrossUseConstantOffsets. It's used to defer
3816/// modifications so that the search phase doesn't have to worry about the data
3817/// structures moving underneath it.
Dan Gohman45774ce2010-02-12 10:34:29 +00003818struct WorkItem {
3819 size_t LUIdx;
3820 int64_t Imm;
3821 const SCEV *OrigReg;
3822
3823 WorkItem(size_t LI, int64_t I, const SCEV *R)
3824 : LUIdx(LI), Imm(I), OrigReg(R) {}
3825
3826 void print(raw_ostream &OS) const;
3827 void dump() const;
3828};
3829
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00003830} // end anonymous namespace
Dan Gohman45774ce2010-02-12 10:34:29 +00003831
3832void WorkItem::print(raw_ostream &OS) const {
3833 OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx
3834 << " , add offset " << Imm;
3835}
3836
Matthias Braun8c209aa2017-01-28 02:02:38 +00003837#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3838LLVM_DUMP_METHOD void WorkItem::dump() const {
Dan Gohman45774ce2010-02-12 10:34:29 +00003839 print(errs()); errs() << '\n';
3840}
Matthias Braun8c209aa2017-01-28 02:02:38 +00003841#endif
Dan Gohman45774ce2010-02-12 10:34:29 +00003842
Sanjoy Das94c4aec2015-08-16 18:22:46 +00003843/// Look for registers which are a constant distance apart and try to form reuse
3844/// opportunities between them.
Dan Gohman45774ce2010-02-12 10:34:29 +00003845void LSRInstance::GenerateCrossUseConstantOffsets() {
3846 // Group the registers by their value without any added constant offset.
3847 typedef std::map<int64_t, const SCEV *> ImmMapTy;
Craig Topper042a3922015-05-25 20:01:18 +00003848 DenseMap<const SCEV *, ImmMapTy> Map;
Dan Gohman45774ce2010-02-12 10:34:29 +00003849 DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
3850 SmallVector<const SCEV *, 8> Sequence;
Craig Topper042a3922015-05-25 20:01:18 +00003851 for (const SCEV *Use : RegUses) {
3852 const SCEV *Reg = Use; // Make a copy for ExtractImmediate to modify.
Dan Gohman45774ce2010-02-12 10:34:29 +00003853 int64_t Imm = ExtractImmediate(Reg, SE);
Craig Topper042a3922015-05-25 20:01:18 +00003854 auto Pair = Map.insert(std::make_pair(Reg, ImmMapTy()));
Dan Gohman45774ce2010-02-12 10:34:29 +00003855 if (Pair.second)
3856 Sequence.push_back(Reg);
Craig Topper042a3922015-05-25 20:01:18 +00003857 Pair.first->second.insert(std::make_pair(Imm, Use));
3858 UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(Use);
Dan Gohman45774ce2010-02-12 10:34:29 +00003859 }
3860
3861 // Now examine each set of registers with the same base value. Build up
3862 // a list of work to do and do the work in a separate step so that we're
3863 // not adding formulae and register counts while we're searching.
Dan Gohman110ed642010-09-01 01:45:53 +00003864 SmallVector<WorkItem, 32> WorkItems;
3865 SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems;
Craig Topper042a3922015-05-25 20:01:18 +00003866 for (const SCEV *Reg : Sequence) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003867 const ImmMapTy &Imms = Map.find(Reg)->second;
3868
Dan Gohman363f8472010-02-12 19:20:37 +00003869 // It's not worthwhile looking for reuse if there's only one offset.
3870 if (Imms.size() == 1)
3871 continue;
3872
Dan Gohman45774ce2010-02-12 10:34:29 +00003873 DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':';
Craig Topper042a3922015-05-25 20:01:18 +00003874 for (const auto &Entry : Imms)
3875 dbgs() << ' ' << Entry.first;
Dan Gohman45774ce2010-02-12 10:34:29 +00003876 dbgs() << '\n');
3877
3878 // Examine each offset.
3879 for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
3880 J != JE; ++J) {
3881 const SCEV *OrigReg = J->second;
3882
3883 int64_t JImm = J->first;
3884 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg);
3885
3886 if (!isa<SCEVConstant>(OrigReg) &&
3887 UsedByIndicesMap[Reg].count() == 1) {
3888 DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n');
3889 continue;
3890 }
3891
3892 // Conservatively examine offsets between this orig reg a few selected
3893 // other orig regs.
3894 ImmMapTy::const_iterator OtherImms[] = {
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00003895 Imms.begin(), std::prev(Imms.end()),
3896 Imms.lower_bound((Imms.begin()->first + std::prev(Imms.end())->first) /
3897 2)
Dan Gohman45774ce2010-02-12 10:34:29 +00003898 };
3899 for (size_t i = 0, e = array_lengthof(OtherImms); i != e; ++i) {
3900 ImmMapTy::const_iterator M = OtherImms[i];
Dan Gohman363f8472010-02-12 19:20:37 +00003901 if (M == J || M == JE) continue;
Dan Gohman45774ce2010-02-12 10:34:29 +00003902
3903 // Compute the difference between the two.
3904 int64_t Imm = (uint64_t)JImm - M->first;
Francis Visoiu Mistrihb52e0362017-05-17 01:07:53 +00003905 for (unsigned LUIdx : UsedByIndices.set_bits())
Dan Gohman45774ce2010-02-12 10:34:29 +00003906 // Make a memo of this use, offset, and register tuple.
David Blaikie70573dc2014-11-19 07:49:26 +00003907 if (UniqueItems.insert(std::make_pair(LUIdx, Imm)).second)
Dan Gohman110ed642010-09-01 01:45:53 +00003908 WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg));
Evan Cheng85a9f432009-11-12 07:35:05 +00003909 }
3910 }
3911 }
3912
Dan Gohman45774ce2010-02-12 10:34:29 +00003913 Map.clear();
3914 Sequence.clear();
3915 UsedByIndicesMap.clear();
Dan Gohman110ed642010-09-01 01:45:53 +00003916 UniqueItems.clear();
Dan Gohman45774ce2010-02-12 10:34:29 +00003917
3918 // Now iterate through the worklist and add new formulae.
Craig Topper042a3922015-05-25 20:01:18 +00003919 for (const WorkItem &WI : WorkItems) {
Dan Gohman45774ce2010-02-12 10:34:29 +00003920 size_t LUIdx = WI.LUIdx;
3921 LSRUse &LU = Uses[LUIdx];
3922 int64_t Imm = WI.Imm;
3923 const SCEV *OrigReg = WI.OrigReg;
3924
Chris Lattner229907c2011-07-18 04:54:35 +00003925 Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType());
Dan Gohman45774ce2010-02-12 10:34:29 +00003926 const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm));
3927 unsigned BitWidth = SE.getTypeSizeInBits(IntTy);
3928
Dan Gohman8b0a4192010-03-01 17:49:51 +00003929 // TODO: Use a more targeted data structure.
Dan Gohman45774ce2010-02-12 10:34:29 +00003930 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00003931 Formula F = LU.Formulae[L];
3932 // FIXME: The code for the scaled and unscaled registers looks
3933 // very similar but slightly different. Investigate if they
3934 // could be merged. That way, we would not have to unscale the
3935 // Formula.
Sanjoy Das302bfd02015-08-16 18:22:43 +00003936 F.unscale();
Dan Gohman45774ce2010-02-12 10:34:29 +00003937 // Use the immediate in the scaled register.
3938 if (F.ScaledReg == OrigReg) {
Chandler Carruth6e479322013-01-07 15:04:40 +00003939 int64_t Offset = (uint64_t)F.BaseOffset + Imm * (uint64_t)F.Scale;
Dan Gohman45774ce2010-02-12 10:34:29 +00003940 // Don't create 50 + reg(-50).
3941 if (F.referencesReg(SE.getSCEV(
Chandler Carruth6e479322013-01-07 15:04:40 +00003942 ConstantInt::get(IntTy, -(uint64_t)Offset))))
Dan Gohman45774ce2010-02-12 10:34:29 +00003943 continue;
3944 Formula NewF = F;
Chandler Carruth6e479322013-01-07 15:04:40 +00003945 NewF.BaseOffset = Offset;
Chandler Carruth26c59fa2013-01-07 14:41:08 +00003946 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
3947 NewF))
Dan Gohman45774ce2010-02-12 10:34:29 +00003948 continue;
3949 NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg);
3950
3951 // If the new scale is a constant in a register, and adding the constant
3952 // value to the immediate would produce a value closer to zero than the
3953 // immediate itself, then the formula isn't worthwhile.
3954 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg))
Sanjoy Das0de2fec2015-12-17 20:28:46 +00003955 if (C->getValue()->isNegative() != (NewF.BaseOffset < 0) &&
3956 (C->getAPInt().abs() * APInt(BitWidth, F.Scale))
3957 .ule(std::abs(NewF.BaseOffset)))
Dan Gohman45774ce2010-02-12 10:34:29 +00003958 continue;
3959
3960 // OK, looks good.
Wei Mi74d5a902017-02-22 21:47:08 +00003961 NewF.canonicalize(*this->L);
Dan Gohman45774ce2010-02-12 10:34:29 +00003962 (void)InsertFormula(LU, LUIdx, NewF);
3963 } else {
3964 // Use the immediate in a base register.
3965 for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) {
3966 const SCEV *BaseReg = F.BaseRegs[N];
3967 if (BaseReg != OrigReg)
3968 continue;
3969 Formula NewF = F;
Chandler Carruth6e479322013-01-07 15:04:40 +00003970 NewF.BaseOffset = (uint64_t)NewF.BaseOffset + Imm;
Chandler Carruth26c59fa2013-01-07 14:41:08 +00003971 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
3972 LU.Kind, LU.AccessTy, NewF)) {
3973 if (!TTI.isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm))
Dan Gohman6136e942011-05-03 00:46:49 +00003974 continue;
3975 NewF = F;
3976 NewF.UnfoldedOffset = (uint64_t)NewF.UnfoldedOffset + Imm;
3977 }
Dan Gohman45774ce2010-02-12 10:34:29 +00003978 NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg);
3979
3980 // If the new formula has a constant in a register, and adding the
3981 // constant value to the immediate would produce a value closer to
3982 // zero than the immediate itself, then the formula isn't worthwhile.
Craig Topper10949ae2015-05-23 08:45:10 +00003983 for (const SCEV *NewReg : NewF.BaseRegs)
3984 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewReg))
Sanjoy Das0de2fec2015-12-17 20:28:46 +00003985 if ((C->getAPInt() + NewF.BaseOffset)
3986 .abs()
3987 .slt(std::abs(NewF.BaseOffset)) &&
3988 (C->getAPInt() + NewF.BaseOffset).countTrailingZeros() >=
3989 countTrailingZeros<uint64_t>(NewF.BaseOffset))
Dan Gohman45774ce2010-02-12 10:34:29 +00003990 goto skip_formula;
3991
3992 // Ok, looks good.
Wei Mi74d5a902017-02-22 21:47:08 +00003993 NewF.canonicalize(*this->L);
Dan Gohman45774ce2010-02-12 10:34:29 +00003994 (void)InsertFormula(LU, LUIdx, NewF);
3995 break;
3996 skip_formula:;
3997 }
3998 }
3999 }
4000 }
Dale Johannesen02cb2bf2009-05-11 17:15:42 +00004001}
4002
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004003/// Generate formulae for each use.
Dan Gohman45774ce2010-02-12 10:34:29 +00004004void
4005LSRInstance::GenerateAllReuseFormulae() {
Dan Gohman521efe62010-02-16 01:42:53 +00004006 // This is split into multiple loops so that hasRegsUsedByUsesOtherThan
Dan Gohman45774ce2010-02-12 10:34:29 +00004007 // queries are more precise.
4008 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4009 LSRUse &LU = Uses[LUIdx];
4010 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4011 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
4012 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4013 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
4014 }
4015 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4016 LSRUse &LU = Uses[LUIdx];
4017 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4018 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
4019 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4020 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
4021 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4022 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
4023 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4024 GenerateScales(LU, LUIdx, LU.Formulae[i]);
Dan Gohman521efe62010-02-16 01:42:53 +00004025 }
4026 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4027 LSRUse &LU = Uses[LUIdx];
Dan Gohman45774ce2010-02-12 10:34:29 +00004028 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4029 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
4030 }
4031
4032 GenerateCrossUseConstantOffsets();
Dan Gohmanbf673e02010-08-29 15:21:38 +00004033
4034 DEBUG(dbgs() << "\n"
4035 "After generating reuse formulae:\n";
4036 print_uses(dbgs()));
Dan Gohman45774ce2010-02-12 10:34:29 +00004037}
4038
Dan Gohman1b61fd92010-10-07 23:43:09 +00004039/// If there are multiple formulae with the same set of registers used
Dan Gohman45774ce2010-02-12 10:34:29 +00004040/// by other uses, pick the best one and delete the others.
4041void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
Dan Gohman5947e162010-10-07 23:52:18 +00004042 DenseSet<const SCEV *> VisitedRegs;
4043 SmallPtrSet<const SCEV *, 16> Regs;
Andrew Trick5df90962011-12-06 03:13:31 +00004044 SmallPtrSet<const SCEV *, 16> LoserRegs;
Dan Gohman45774ce2010-02-12 10:34:29 +00004045#ifndef NDEBUG
Dan Gohman4c4043c2010-05-20 20:05:31 +00004046 bool ChangedFormulae = false;
Dan Gohman45774ce2010-02-12 10:34:29 +00004047#endif
4048
4049 // Collect the best formula for each unique set of shared registers. This
4050 // is reset for each use.
Preston Gurd25c3b6a2013-02-01 20:41:27 +00004051 typedef DenseMap<SmallVector<const SCEV *, 4>, size_t, UniquifierDenseMapInfo>
Dan Gohman45774ce2010-02-12 10:34:29 +00004052 BestFormulaeTy;
4053 BestFormulaeTy BestFormulae;
4054
4055 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4056 LSRUse &LU = Uses[LUIdx];
Dan Gohmanab5fb7f2010-05-20 19:44:23 +00004057 DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n');
Dan Gohman45774ce2010-02-12 10:34:29 +00004058
Dan Gohman4cf99b52010-05-18 23:42:37 +00004059 bool Any = false;
Dan Gohman45774ce2010-02-12 10:34:29 +00004060 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4061 FIdx != NumForms; ++FIdx) {
4062 Formula &F = LU.Formulae[FIdx];
4063
Andrew Trick5df90962011-12-06 03:13:31 +00004064 // Some formulas are instant losers. For example, they may depend on
4065 // nonexistent AddRecs from other loops. These need to be filtered
4066 // immediately, otherwise heuristics could choose them over others leading
4067 // to an unsatisfactory solution. Passing LoserRegs into RateFormula here
4068 // avoids the need to recompute this information across formulae using the
4069 // same bad AddRec. Passing LoserRegs is also essential unless we remove
4070 // the corresponding bad register from the Regs set.
4071 Cost CostF;
4072 Regs.clear();
Jonas Paulsson7a794222016-08-17 13:24:19 +00004073 CostF.RateFormula(TTI, F, Regs, VisitedRegs, L, SE, DT, LU, &LoserRegs);
Andrew Trick5df90962011-12-06 03:13:31 +00004074 if (CostF.isLoser()) {
4075 // During initial formula generation, undesirable formulae are generated
4076 // by uses within other loops that have some non-trivial address mode or
4077 // use the postinc form of the IV. LSR needs to provide these formulae
4078 // as the basis of rediscovering the desired formula that uses an AddRec
4079 // corresponding to the existing phi. Once all formulae have been
4080 // generated, these initial losers may be pruned.
4081 DEBUG(dbgs() << " Filtering loser "; F.print(dbgs());
4082 dbgs() << "\n");
Dan Gohman45774ce2010-02-12 10:34:29 +00004083 }
Andrew Trick5df90962011-12-06 03:13:31 +00004084 else {
Preston Gurd25c3b6a2013-02-01 20:41:27 +00004085 SmallVector<const SCEV *, 4> Key;
Craig Topper77b99412015-05-23 08:01:41 +00004086 for (const SCEV *Reg : F.BaseRegs) {
Andrew Trick5df90962011-12-06 03:13:31 +00004087 if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
4088 Key.push_back(Reg);
4089 }
4090 if (F.ScaledReg &&
4091 RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx))
4092 Key.push_back(F.ScaledReg);
4093 // Unstable sort by host order ok, because this is only used for
4094 // uniquifying.
4095 std::sort(Key.begin(), Key.end());
Dan Gohman45774ce2010-02-12 10:34:29 +00004096
Andrew Trick5df90962011-12-06 03:13:31 +00004097 std::pair<BestFormulaeTy::const_iterator, bool> P =
4098 BestFormulae.insert(std::make_pair(Key, FIdx));
4099 if (P.second)
4100 continue;
4101
Dan Gohman45774ce2010-02-12 10:34:29 +00004102 Formula &Best = LU.Formulae[P.first->second];
Dan Gohman5947e162010-10-07 23:52:18 +00004103
Dan Gohman5947e162010-10-07 23:52:18 +00004104 Cost CostBest;
Dan Gohman5947e162010-10-07 23:52:18 +00004105 Regs.clear();
Jonas Paulsson7a794222016-08-17 13:24:19 +00004106 CostBest.RateFormula(TTI, Best, Regs, VisitedRegs, L, SE, DT, LU);
Dan Gohman5947e162010-10-07 23:52:18 +00004107 if (CostF < CostBest)
Dan Gohman45774ce2010-02-12 10:34:29 +00004108 std::swap(F, Best);
Dan Gohman8aca7ef2010-05-18 22:37:37 +00004109 DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
Dan Gohman45774ce2010-02-12 10:34:29 +00004110 dbgs() << "\n"
Dan Gohman8aca7ef2010-05-18 22:37:37 +00004111 " in favor of formula "; Best.print(dbgs());
Dan Gohman45774ce2010-02-12 10:34:29 +00004112 dbgs() << '\n');
Dan Gohman45774ce2010-02-12 10:34:29 +00004113 }
Andrew Trick5df90962011-12-06 03:13:31 +00004114#ifndef NDEBUG
4115 ChangedFormulae = true;
4116#endif
4117 LU.DeleteFormula(F);
4118 --FIdx;
4119 --NumForms;
4120 Any = true;
Dan Gohmand0800242010-05-07 23:36:59 +00004121 }
4122
Dan Gohmanbeebef42010-05-18 23:55:57 +00004123 // Now that we've filtered out some formulae, recompute the Regs set.
Dan Gohman4cf99b52010-05-18 23:42:37 +00004124 if (Any)
4125 LU.RecomputeRegs(LUIdx, RegUses);
Dan Gohmand0800242010-05-07 23:36:59 +00004126
4127 // Reset this to prepare for the next use.
Dan Gohman45774ce2010-02-12 10:34:29 +00004128 BestFormulae.clear();
4129 }
4130
Dan Gohman4c4043c2010-05-20 20:05:31 +00004131 DEBUG(if (ChangedFormulae) {
Dan Gohman5b18f032010-02-13 02:06:02 +00004132 dbgs() << "\n"
4133 "After filtering out undesirable candidates:\n";
Dan Gohman45774ce2010-02-12 10:34:29 +00004134 print_uses(dbgs());
4135 });
4136}
4137
Dan Gohmana4eca052010-05-18 22:51:59 +00004138// This is a rough guess that seems to work fairly well.
4139static const size_t ComplexityLimit = UINT16_MAX;
4140
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004141/// Estimate the worst-case number of solutions the solver might have to
4142/// consider. It almost never considers this many solutions because it prune the
4143/// search space, but the pruning isn't always sufficient.
Dan Gohmana4eca052010-05-18 22:51:59 +00004144size_t LSRInstance::EstimateSearchSpaceComplexity() const {
Dan Gohman49d638b2010-10-07 23:37:58 +00004145 size_t Power = 1;
Craig Topper10949ae2015-05-23 08:45:10 +00004146 for (const LSRUse &LU : Uses) {
4147 size_t FSize = LU.Formulae.size();
Dan Gohmana4eca052010-05-18 22:51:59 +00004148 if (FSize >= ComplexityLimit) {
4149 Power = ComplexityLimit;
4150 break;
4151 }
4152 Power *= FSize;
4153 if (Power >= ComplexityLimit)
4154 break;
4155 }
4156 return Power;
4157}
4158
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004159/// When one formula uses a superset of the registers of another formula, it
4160/// won't help reduce register pressure (though it may not necessarily hurt
4161/// register pressure); remove it to simplify the system.
Dan Gohmane9e08732010-08-29 16:09:42 +00004162void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
Dan Gohman20fab452010-05-19 23:43:12 +00004163 if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
4164 DEBUG(dbgs() << "The search space is too complex.\n");
4165
4166 DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "
4167 "which use a superset of registers used by other "
4168 "formulae.\n");
4169
4170 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4171 LSRUse &LU = Uses[LUIdx];
4172 bool Any = false;
4173 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4174 Formula &F = LU.Formulae[i];
Dan Gohman8ec018c2010-05-20 20:00:41 +00004175 // Look for a formula with a constant or GV in a register. If the use
4176 // also has a formula with that same value in an immediate field,
4177 // delete the one that uses a register.
Dan Gohman20fab452010-05-19 23:43:12 +00004178 for (SmallVectorImpl<const SCEV *>::const_iterator
4179 I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) {
4180 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) {
4181 Formula NewF = F;
Chandler Carruth6e479322013-01-07 15:04:40 +00004182 NewF.BaseOffset += C->getValue()->getSExtValue();
Dan Gohman20fab452010-05-19 23:43:12 +00004183 NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
4184 (I - F.BaseRegs.begin()));
4185 if (LU.HasFormulaWithSameRegs(NewF)) {
4186 DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
4187 LU.DeleteFormula(F);
4188 --i;
4189 --e;
4190 Any = true;
4191 break;
4192 }
4193 } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) {
4194 if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue()))
Chandler Carruth6e479322013-01-07 15:04:40 +00004195 if (!F.BaseGV) {
Dan Gohman20fab452010-05-19 23:43:12 +00004196 Formula NewF = F;
Chandler Carruth6e479322013-01-07 15:04:40 +00004197 NewF.BaseGV = GV;
Dan Gohman20fab452010-05-19 23:43:12 +00004198 NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
4199 (I - F.BaseRegs.begin()));
4200 if (LU.HasFormulaWithSameRegs(NewF)) {
4201 DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4202 dbgs() << '\n');
4203 LU.DeleteFormula(F);
4204 --i;
4205 --e;
4206 Any = true;
4207 break;
4208 }
4209 }
4210 }
4211 }
4212 }
4213 if (Any)
4214 LU.RecomputeRegs(LUIdx, RegUses);
4215 }
4216
4217 DEBUG(dbgs() << "After pre-selection:\n";
4218 print_uses(dbgs()));
4219 }
Dan Gohmane9e08732010-08-29 16:09:42 +00004220}
Dan Gohman20fab452010-05-19 23:43:12 +00004221
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004222/// When there are many registers for expressions like A, A+1, A+2, etc.,
4223/// allocate a single register for them.
Dan Gohmane9e08732010-08-29 16:09:42 +00004224void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
Jakub Staszak11bd8352013-02-16 16:08:15 +00004225 if (EstimateSearchSpaceComplexity() < ComplexityLimit)
4226 return;
Dan Gohman20fab452010-05-19 23:43:12 +00004227
Jakub Staszak11bd8352013-02-16 16:08:15 +00004228 DEBUG(dbgs() << "The search space is too complex.\n"
4229 "Narrowing the search space by assuming that uses separated "
4230 "by a constant offset will use the same registers.\n");
Dan Gohman20fab452010-05-19 23:43:12 +00004231
Jakub Staszak11bd8352013-02-16 16:08:15 +00004232 // This is especially useful for unrolled loops.
Dan Gohman8ec018c2010-05-20 20:00:41 +00004233
Jakub Staszak11bd8352013-02-16 16:08:15 +00004234 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4235 LSRUse &LU = Uses[LUIdx];
Craig Topper77b99412015-05-23 08:01:41 +00004236 for (const Formula &F : LU.Formulae) {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00004237 if (F.BaseOffset == 0 || (F.Scale != 0 && F.Scale != 1))
Jakub Staszak11bd8352013-02-16 16:08:15 +00004238 continue;
Dan Gohman20fab452010-05-19 23:43:12 +00004239
Jakub Staszak11bd8352013-02-16 16:08:15 +00004240 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
4241 if (!LUThatHas)
4242 continue;
Dan Gohman20fab452010-05-19 23:43:12 +00004243
Jakub Staszak11bd8352013-02-16 16:08:15 +00004244 if (!reconcileNewOffset(*LUThatHas, F.BaseOffset, /*HasBaseReg=*/ false,
4245 LU.Kind, LU.AccessTy))
4246 continue;
Dan Gohman110ed642010-09-01 01:45:53 +00004247
Jakub Staszak11bd8352013-02-16 16:08:15 +00004248 DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
Dan Gohman2fd85d72010-10-08 19:33:26 +00004249
Jakub Staszak11bd8352013-02-16 16:08:15 +00004250 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
4251
Jonas Paulsson7a794222016-08-17 13:24:19 +00004252 // Transfer the fixups of LU to LUThatHas.
4253 for (LSRFixup &Fixup : LU.Fixups) {
4254 Fixup.Offset += F.BaseOffset;
4255 LUThatHas->pushFixup(Fixup);
4256 DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << '\n');
Jakub Staszak11bd8352013-02-16 16:08:15 +00004257 }
Jonas Paulsson7a794222016-08-17 13:24:19 +00004258
Jakub Staszak11bd8352013-02-16 16:08:15 +00004259 // Delete formulae from the new use which are no longer legal.
4260 bool Any = false;
4261 for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) {
4262 Formula &F = LUThatHas->Formulae[i];
4263 if (!isLegalUse(TTI, LUThatHas->MinOffset, LUThatHas->MaxOffset,
4264 LUThatHas->Kind, LUThatHas->AccessTy, F)) {
4265 DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4266 dbgs() << '\n');
4267 LUThatHas->DeleteFormula(F);
4268 --i;
4269 --e;
4270 Any = true;
Dan Gohman20fab452010-05-19 23:43:12 +00004271 }
4272 }
Dan Gohman20fab452010-05-19 23:43:12 +00004273
Jakub Staszak11bd8352013-02-16 16:08:15 +00004274 if (Any)
4275 LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses);
4276
4277 // Delete the old use.
4278 DeleteUse(LU, LUIdx);
4279 --LUIdx;
4280 --NumUses;
4281 break;
4282 }
Dan Gohman20fab452010-05-19 23:43:12 +00004283 }
Jakub Staszak11bd8352013-02-16 16:08:15 +00004284
4285 DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
Dan Gohmane9e08732010-08-29 16:09:42 +00004286}
Dan Gohman20fab452010-05-19 23:43:12 +00004287
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004288/// Call FilterOutUndesirableDedicatedRegisters again, if necessary, now that
Dan Gohman002ff892010-08-29 16:39:22 +00004289/// we've done more filtering, as it may be able to find more formulae to
4290/// eliminate.
4291void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){
4292 if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
4293 DEBUG(dbgs() << "The search space is too complex.\n");
4294
4295 DEBUG(dbgs() << "Narrowing the search space by re-filtering out "
4296 "undesirable dedicated registers.\n");
4297
4298 FilterOutUndesirableDedicatedRegisters();
4299
4300 DEBUG(dbgs() << "After pre-selection:\n";
4301 print_uses(dbgs()));
4302 }
4303}
4304
Evgeny Stupachenko9909872e302017-02-21 07:34:40 +00004305/// The function delete formulas with high registers number expectation.
4306/// Assuming we don't know the value of each formula (already delete
4307/// all inefficient), generate probability of not selecting for each
4308/// register.
4309/// For example,
4310/// Use1:
4311/// reg(a) + reg({0,+,1})
4312/// reg(a) + reg({-1,+,1}) + 1
4313/// reg({a,+,1})
4314/// Use2:
4315/// reg(b) + reg({0,+,1})
4316/// reg(b) + reg({-1,+,1}) + 1
4317/// reg({b,+,1})
4318/// Use3:
4319/// reg(c) + reg(b) + reg({0,+,1})
4320/// reg(c) + reg({b,+,1})
4321///
4322/// Probability of not selecting
4323/// Use1 Use2 Use3
4324/// reg(a) (1/3) * 1 * 1
4325/// reg(b) 1 * (1/3) * (1/2)
4326/// reg({0,+,1}) (2/3) * (2/3) * (1/2)
4327/// reg({-1,+,1}) (2/3) * (2/3) * 1
4328/// reg({a,+,1}) (2/3) * 1 * 1
4329/// reg({b,+,1}) 1 * (2/3) * (2/3)
4330/// reg(c) 1 * 1 * 0
4331///
4332/// Now count registers number mathematical expectation for each formula:
4333/// Note that for each use we exclude probability if not selecting for the use.
4334/// For example for Use1 probability for reg(a) would be just 1 * 1 (excluding
4335/// probabilty 1/3 of not selecting for Use1).
4336/// Use1:
4337/// reg(a) + reg({0,+,1}) 1 + 1/3 -- to be deleted
4338/// reg(a) + reg({-1,+,1}) + 1 1 + 4/9 -- to be deleted
4339/// reg({a,+,1}) 1
4340/// Use2:
4341/// reg(b) + reg({0,+,1}) 1/2 + 1/3 -- to be deleted
4342/// reg(b) + reg({-1,+,1}) + 1 1/2 + 2/3 -- to be deleted
4343/// reg({b,+,1}) 2/3
4344/// Use3:
4345/// reg(c) + reg(b) + reg({0,+,1}) 1 + 1/3 + 4/9 -- to be deleted
4346/// reg(c) + reg({b,+,1}) 1 + 2/3
4347
4348void LSRInstance::NarrowSearchSpaceByDeletingCostlyFormulas() {
4349 if (EstimateSearchSpaceComplexity() < ComplexityLimit)
4350 return;
4351 // Ok, we have too many of formulae on our hands to conveniently handle.
4352 // Use a rough heuristic to thin out the list.
4353
4354 // Set of Regs wich will be 100% used in final solution.
4355 // Used in each formula of a solution (in example above this is reg(c)).
4356 // We can skip them in calculations.
4357 SmallPtrSet<const SCEV *, 4> UniqRegs;
4358 DEBUG(dbgs() << "The search space is too complex.\n");
4359
4360 // Map each register to probability of not selecting
4361 DenseMap <const SCEV *, float> RegNumMap;
4362 for (const SCEV *Reg : RegUses) {
4363 if (UniqRegs.count(Reg))
4364 continue;
4365 float PNotSel = 1;
4366 for (const LSRUse &LU : Uses) {
4367 if (!LU.Regs.count(Reg))
4368 continue;
4369 float P = LU.getNotSelectedProbability(Reg);
4370 if (P != 0.0)
4371 PNotSel *= P;
4372 else
4373 UniqRegs.insert(Reg);
4374 }
4375 RegNumMap.insert(std::make_pair(Reg, PNotSel));
4376 }
4377
4378 DEBUG(dbgs() << "Narrowing the search space by deleting costly formulas\n");
4379
4380 // Delete formulas where registers number expectation is high.
4381 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4382 LSRUse &LU = Uses[LUIdx];
4383 // If nothing to delete - continue.
4384 if (LU.Formulae.size() < 2)
4385 continue;
4386 // This is temporary solution to test performance. Float should be
4387 // replaced with round independent type (based on integers) to avoid
4388 // different results for different target builds.
4389 float FMinRegNum = LU.Formulae[0].getNumRegs();
4390 float FMinARegNum = LU.Formulae[0].getNumRegs();
4391 size_t MinIdx = 0;
4392 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4393 Formula &F = LU.Formulae[i];
4394 float FRegNum = 0;
4395 float FARegNum = 0;
4396 for (const SCEV *BaseReg : F.BaseRegs) {
4397 if (UniqRegs.count(BaseReg))
4398 continue;
4399 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4400 if (isa<SCEVAddRecExpr>(BaseReg))
4401 FARegNum +=
4402 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
4403 }
4404 if (const SCEV *ScaledReg = F.ScaledReg) {
4405 if (!UniqRegs.count(ScaledReg)) {
4406 FRegNum +=
4407 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4408 if (isa<SCEVAddRecExpr>(ScaledReg))
4409 FARegNum +=
4410 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
4411 }
4412 }
4413 if (FMinRegNum > FRegNum ||
4414 (FMinRegNum == FRegNum && FMinARegNum > FARegNum)) {
4415 FMinRegNum = FRegNum;
4416 FMinARegNum = FARegNum;
4417 MinIdx = i;
4418 }
4419 }
4420 DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
4421 dbgs() << " with min reg num " << FMinRegNum << '\n');
4422 if (MinIdx != 0)
4423 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]);
4424 while (LU.Formulae.size() != 1) {
4425 DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
4426 dbgs() << '\n');
4427 LU.Formulae.pop_back();
4428 }
4429 LU.RecomputeRegs(LUIdx, RegUses);
4430 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
4431 Formula &F = LU.Formulae[0];
4432 DEBUG(dbgs() << " Leaving only "; F.print(dbgs()); dbgs() << '\n');
4433 // When we choose the formula, the regs become unique.
4434 UniqRegs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
4435 if (F.ScaledReg)
4436 UniqRegs.insert(F.ScaledReg);
4437 }
4438 DEBUG(dbgs() << "After pre-selection:\n";
4439 print_uses(dbgs()));
4440}
4441
4442
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004443/// Pick a register which seems likely to be profitable, and then in any use
4444/// which has any reference to that register, delete all formulae which do not
4445/// reference that register.
Dan Gohmane9e08732010-08-29 16:09:42 +00004446void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
Dan Gohmana4ca28a2010-05-20 20:52:00 +00004447 // With all other options exhausted, loop until the system is simple
4448 // enough to handle.
Dan Gohman45774ce2010-02-12 10:34:29 +00004449 SmallPtrSet<const SCEV *, 4> Taken;
Dan Gohmana4eca052010-05-18 22:51:59 +00004450 while (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
Dan Gohman45774ce2010-02-12 10:34:29 +00004451 // Ok, we have too many of formulae on our hands to conveniently handle.
4452 // Use a rough heuristic to thin out the list.
Dan Gohman63e90152010-05-18 22:41:32 +00004453 DEBUG(dbgs() << "The search space is too complex.\n");
Dan Gohman45774ce2010-02-12 10:34:29 +00004454
4455 // Pick the register which is used by the most LSRUses, which is likely
4456 // to be a good reuse register candidate.
Craig Topperf40110f2014-04-25 05:29:35 +00004457 const SCEV *Best = nullptr;
Dan Gohman45774ce2010-02-12 10:34:29 +00004458 unsigned BestNum = 0;
Craig Topper77b99412015-05-23 08:01:41 +00004459 for (const SCEV *Reg : RegUses) {
Dan Gohman45774ce2010-02-12 10:34:29 +00004460 if (Taken.count(Reg))
4461 continue;
Evgeny Stupachenko0c4300f2016-11-30 22:23:51 +00004462 if (!Best) {
Dan Gohman45774ce2010-02-12 10:34:29 +00004463 Best = Reg;
Evgeny Stupachenko0c4300f2016-11-30 22:23:51 +00004464 BestNum = RegUses.getUsedByIndices(Reg).count();
4465 } else {
Dan Gohman45774ce2010-02-12 10:34:29 +00004466 unsigned Count = RegUses.getUsedByIndices(Reg).count();
4467 if (Count > BestNum) {
4468 Best = Reg;
4469 BestNum = Count;
4470 }
4471 }
4472 }
4473
4474 DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
Dan Gohman8b0a4192010-03-01 17:49:51 +00004475 << " will yield profitable reuse.\n");
Dan Gohman45774ce2010-02-12 10:34:29 +00004476 Taken.insert(Best);
4477
4478 // In any use with formulae which references this register, delete formulae
4479 // which don't reference it.
Dan Gohman4cf99b52010-05-18 23:42:37 +00004480 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4481 LSRUse &LU = Uses[LUIdx];
Dan Gohman45774ce2010-02-12 10:34:29 +00004482 if (!LU.Regs.count(Best)) continue;
4483
Dan Gohman4cf99b52010-05-18 23:42:37 +00004484 bool Any = false;
Dan Gohman45774ce2010-02-12 10:34:29 +00004485 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4486 Formula &F = LU.Formulae[i];
4487 if (!F.referencesReg(Best)) {
4488 DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
Dan Gohmanf1c7b1b2010-05-18 22:39:15 +00004489 LU.DeleteFormula(F);
Dan Gohman45774ce2010-02-12 10:34:29 +00004490 --e;
4491 --i;
Dan Gohman4cf99b52010-05-18 23:42:37 +00004492 Any = true;
Dan Gohmand0800242010-05-07 23:36:59 +00004493 assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?");
Dan Gohman45774ce2010-02-12 10:34:29 +00004494 continue;
4495 }
Dan Gohman45774ce2010-02-12 10:34:29 +00004496 }
Dan Gohman4cf99b52010-05-18 23:42:37 +00004497
4498 if (Any)
4499 LU.RecomputeRegs(LUIdx, RegUses);
Dan Gohman45774ce2010-02-12 10:34:29 +00004500 }
4501
4502 DEBUG(dbgs() << "After pre-selection:\n";
4503 print_uses(dbgs()));
4504 }
4505}
4506
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004507/// If there are an extraordinary number of formulae to choose from, use some
4508/// rough heuristics to prune down the number of formulae. This keeps the main
4509/// solver from taking an extraordinary amount of time in some worst-case
4510/// scenarios.
Dan Gohmane9e08732010-08-29 16:09:42 +00004511void LSRInstance::NarrowSearchSpaceUsingHeuristics() {
4512 NarrowSearchSpaceByDetectingSupersets();
4513 NarrowSearchSpaceByCollapsingUnrolledCode();
Dan Gohman002ff892010-08-29 16:39:22 +00004514 NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
Evgeny Stupachenko9909872e302017-02-21 07:34:40 +00004515 if (LSRExpNarrow)
4516 NarrowSearchSpaceByDeletingCostlyFormulas();
4517 else
4518 NarrowSearchSpaceByPickingWinnerRegs();
Dan Gohmane9e08732010-08-29 16:09:42 +00004519}
4520
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004521/// This is the recursive solver.
Dan Gohman45774ce2010-02-12 10:34:29 +00004522void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
4523 Cost &SolutionCost,
4524 SmallVectorImpl<const Formula *> &Workspace,
4525 const Cost &CurCost,
4526 const SmallPtrSet<const SCEV *, 16> &CurRegs,
4527 DenseSet<const SCEV *> &VisitedRegs) const {
4528 // Some ideas:
4529 // - prune more:
4530 // - use more aggressive filtering
4531 // - sort the formula so that the most profitable solutions are found first
4532 // - sort the uses too
4533 // - search faster:
Dan Gohman8b0a4192010-03-01 17:49:51 +00004534 // - don't compute a cost, and then compare. compare while computing a cost
Dan Gohman45774ce2010-02-12 10:34:29 +00004535 // and bail early.
4536 // - track register sets with SmallBitVector
4537
4538 const LSRUse &LU = Uses[Workspace.size()];
4539
4540 // If this use references any register that's already a part of the
4541 // in-progress solution, consider it a requirement that a formula must
4542 // reference that register in order to be considered. This prunes out
4543 // unprofitable searching.
4544 SmallSetVector<const SCEV *, 4> ReqRegs;
Craig Topper46276792014-08-24 23:23:06 +00004545 for (const SCEV *S : CurRegs)
4546 if (LU.Regs.count(S))
4547 ReqRegs.insert(S);
Dan Gohman45774ce2010-02-12 10:34:29 +00004548
4549 SmallPtrSet<const SCEV *, 16> NewRegs;
4550 Cost NewCost;
Craig Topper77b99412015-05-23 08:01:41 +00004551 for (const Formula &F : LU.Formulae) {
Adam Nemetdeab6f92014-04-29 18:25:28 +00004552 // Ignore formulae which may not be ideal in terms of register reuse of
4553 // ReqRegs. The formula should use all required registers before
4554 // introducing new ones.
4555 int NumReqRegsToFind = std::min(F.getNumRegs(), ReqRegs.size());
Craig Topper77b99412015-05-23 08:01:41 +00004556 for (const SCEV *Reg : ReqRegs) {
Adam Nemetdeab6f92014-04-29 18:25:28 +00004557 if ((F.ScaledReg && F.ScaledReg == Reg) ||
David Majnemer0d955d02016-08-11 22:21:41 +00004558 is_contained(F.BaseRegs, Reg)) {
Adam Nemetdeab6f92014-04-29 18:25:28 +00004559 --NumReqRegsToFind;
4560 if (NumReqRegsToFind == 0)
4561 break;
Andrew Tricke3502cb2012-03-22 22:42:51 +00004562 }
Dan Gohman45774ce2010-02-12 10:34:29 +00004563 }
Adam Nemetdeab6f92014-04-29 18:25:28 +00004564 if (NumReqRegsToFind != 0) {
Andrew Tricke3502cb2012-03-22 22:42:51 +00004565 // If none of the formulae satisfied the required registers, then we could
4566 // clear ReqRegs and try again. Currently, we simply give up in this case.
4567 continue;
4568 }
Dan Gohman45774ce2010-02-12 10:34:29 +00004569
4570 // Evaluate the cost of the current formula. If it's already worse than
4571 // the current best, prune the search at that point.
4572 NewCost = CurCost;
4573 NewRegs = CurRegs;
Jonas Paulsson7a794222016-08-17 13:24:19 +00004574 NewCost.RateFormula(TTI, F, NewRegs, VisitedRegs, L, SE, DT, LU);
Dan Gohman45774ce2010-02-12 10:34:29 +00004575 if (NewCost < SolutionCost) {
4576 Workspace.push_back(&F);
4577 if (Workspace.size() != Uses.size()) {
4578 SolveRecurse(Solution, SolutionCost, Workspace, NewCost,
4579 NewRegs, VisitedRegs);
4580 if (F.getNumRegs() == 1 && Workspace.size() == 1)
4581 VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]);
4582 } else {
4583 DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());
Andrew Trick4dc3eff2012-01-09 18:58:16 +00004584 dbgs() << ".\n Regs:";
Craig Topper46276792014-08-24 23:23:06 +00004585 for (const SCEV *S : NewRegs)
4586 dbgs() << ' ' << *S;
Dan Gohman45774ce2010-02-12 10:34:29 +00004587 dbgs() << '\n');
4588
4589 SolutionCost = NewCost;
4590 Solution = Workspace;
4591 }
4592 Workspace.pop_back();
4593 }
Dan Gohman5b18f032010-02-13 02:06:02 +00004594 }
Dan Gohman45774ce2010-02-12 10:34:29 +00004595}
4596
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004597/// Choose one formula from each use. Return the results in the given Solution
4598/// vector.
Dan Gohman45774ce2010-02-12 10:34:29 +00004599void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
4600 SmallVector<const Formula *, 8> Workspace;
4601 Cost SolutionCost;
Tim Northoverbc6659c2014-01-22 13:27:00 +00004602 SolutionCost.Lose();
Dan Gohman45774ce2010-02-12 10:34:29 +00004603 Cost CurCost;
4604 SmallPtrSet<const SCEV *, 16> CurRegs;
4605 DenseSet<const SCEV *> VisitedRegs;
4606 Workspace.reserve(Uses.size());
4607
Dan Gohman8ec018c2010-05-20 20:00:41 +00004608 // SolveRecurse does all the work.
Dan Gohman45774ce2010-02-12 10:34:29 +00004609 SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
4610 CurRegs, VisitedRegs);
Andrew Trick58124392011-09-27 00:44:14 +00004611 if (Solution.empty()) {
4612 DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
4613 return;
4614 }
Dan Gohman45774ce2010-02-12 10:34:29 +00004615
4616 // Ok, we've now made all our decisions.
4617 DEBUG(dbgs() << "\n"
4618 "The chosen solution requires "; SolutionCost.print(dbgs());
4619 dbgs() << ":\n";
4620 for (size_t i = 0, e = Uses.size(); i != e; ++i) {
4621 dbgs() << " ";
4622 Uses[i].print(dbgs());
4623 dbgs() << "\n"
4624 " ";
4625 Solution[i]->print(dbgs());
4626 dbgs() << '\n';
4627 });
Dan Gohman6295f2e2010-05-20 20:59:23 +00004628
4629 assert(Solution.size() == Uses.size() && "Malformed solution!");
Dan Gohman45774ce2010-02-12 10:34:29 +00004630}
4631
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004632/// Helper for AdjustInsertPositionForExpand. Climb up the dominator tree far as
4633/// we can go while still being dominated by the input positions. This helps
4634/// canonicalize the insert position, which encourages sharing.
Dan Gohman607e02b2010-04-09 22:07:05 +00004635BasicBlock::iterator
4636LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
4637 const SmallVectorImpl<Instruction *> &Inputs)
4638 const {
Geoff Berry43e51602016-06-06 19:10:46 +00004639 Instruction *Tentative = &*IP;
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00004640 while (true) {
Geoff Berry43e51602016-06-06 19:10:46 +00004641 bool AllDominate = true;
4642 Instruction *BetterPos = nullptr;
4643 // Don't bother attempting to insert before a catchswitch, their basic block
4644 // cannot have other non-PHI instructions.
4645 if (isa<CatchSwitchInst>(Tentative))
4646 return IP;
4647
4648 for (Instruction *Inst : Inputs) {
4649 if (Inst == Tentative || !DT.dominates(Inst, Tentative)) {
4650 AllDominate = false;
4651 break;
4652 }
4653 // Attempt to find an insert position in the middle of the block,
4654 // instead of at the end, so that it can be used for other expansions.
4655 if (Tentative->getParent() == Inst->getParent() &&
4656 (!BetterPos || !DT.dominates(Inst, BetterPos)))
4657 BetterPos = &*std::next(BasicBlock::iterator(Inst));
4658 }
4659 if (!AllDominate)
4660 break;
4661 if (BetterPos)
4662 IP = BetterPos->getIterator();
4663 else
4664 IP = Tentative->getIterator();
4665
Dan Gohman607e02b2010-04-09 22:07:05 +00004666 const Loop *IPLoop = LI.getLoopFor(IP->getParent());
4667 unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0;
4668
4669 BasicBlock *IDom;
Dan Gohman8ce95cc2010-05-20 20:00:25 +00004670 for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) {
Dan Gohman9b48b852010-05-20 22:46:54 +00004671 if (!Rung) return IP;
Dan Gohman8ce95cc2010-05-20 20:00:25 +00004672 Rung = Rung->getIDom();
4673 if (!Rung) return IP;
4674 IDom = Rung->getBlock();
Dan Gohman607e02b2010-04-09 22:07:05 +00004675
4676 // Don't climb into a loop though.
4677 const Loop *IDomLoop = LI.getLoopFor(IDom);
4678 unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0;
4679 if (IDomDepth <= IPLoopDepth &&
4680 (IDomDepth != IPLoopDepth || IDomLoop == IPLoop))
4681 break;
4682 }
4683
Geoff Berry43e51602016-06-06 19:10:46 +00004684 Tentative = IDom->getTerminator();
Dan Gohman607e02b2010-04-09 22:07:05 +00004685 }
4686
4687 return IP;
4688}
4689
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004690/// Determine an input position which will be dominated by the operands and
4691/// which will dominate the result.
Dan Gohmand2df6432010-04-09 02:00:38 +00004692BasicBlock::iterator
Andrew Trickc908b432012-01-20 07:41:13 +00004693LSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator LowestIP,
Dan Gohman607e02b2010-04-09 22:07:05 +00004694 const LSRFixup &LF,
Andrew Trickc908b432012-01-20 07:41:13 +00004695 const LSRUse &LU,
4696 SCEVExpander &Rewriter) const {
Dan Gohmand2df6432010-04-09 02:00:38 +00004697 // Collect some instructions which must be dominated by the
Dan Gohmand006ab92010-04-07 22:27:08 +00004698 // expanding replacement. These must be dominated by any operands that
Dan Gohman45774ce2010-02-12 10:34:29 +00004699 // will be required in the expansion.
4700 SmallVector<Instruction *, 4> Inputs;
4701 if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace))
4702 Inputs.push_back(I);
4703 if (LU.Kind == LSRUse::ICmpZero)
4704 if (Instruction *I =
4705 dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1)))
4706 Inputs.push_back(I);
Dan Gohmand006ab92010-04-07 22:27:08 +00004707 if (LF.PostIncLoops.count(L)) {
4708 if (LF.isUseFullyOutsideLoop(L))
Dan Gohman52f55632010-03-02 01:59:21 +00004709 Inputs.push_back(L->getLoopLatch()->getTerminator());
4710 else
4711 Inputs.push_back(IVIncInsertPos);
4712 }
Dan Gohman45065392010-04-08 05:57:57 +00004713 // The expansion must also be dominated by the increment positions of any
4714 // loops it for which it is using post-inc mode.
Craig Topper77b99412015-05-23 08:01:41 +00004715 for (const Loop *PIL : LF.PostIncLoops) {
Dan Gohman45065392010-04-08 05:57:57 +00004716 if (PIL == L) continue;
4717
Dan Gohman607e02b2010-04-09 22:07:05 +00004718 // Be dominated by the loop exit.
Dan Gohman45065392010-04-08 05:57:57 +00004719 SmallVector<BasicBlock *, 4> ExitingBlocks;
4720 PIL->getExitingBlocks(ExitingBlocks);
4721 if (!ExitingBlocks.empty()) {
4722 BasicBlock *BB = ExitingBlocks[0];
4723 for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i)
4724 BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]);
4725 Inputs.push_back(BB->getTerminator());
4726 }
4727 }
Dan Gohman45774ce2010-02-12 10:34:29 +00004728
David Majnemerba275f92015-08-19 19:54:02 +00004729 assert(!isa<PHINode>(LowestIP) && !LowestIP->isEHPad()
Andrew Trickc908b432012-01-20 07:41:13 +00004730 && !isa<DbgInfoIntrinsic>(LowestIP) &&
4731 "Insertion point must be a normal instruction");
4732
Dan Gohman45774ce2010-02-12 10:34:29 +00004733 // Then, climb up the immediate dominator tree as far as we can go while
4734 // still being dominated by the input positions.
Andrew Trickc908b432012-01-20 07:41:13 +00004735 BasicBlock::iterator IP = HoistInsertPosition(LowestIP, Inputs);
Dan Gohmand2df6432010-04-09 02:00:38 +00004736
4737 // Don't insert instructions before PHI nodes.
Dan Gohman45774ce2010-02-12 10:34:29 +00004738 while (isa<PHINode>(IP)) ++IP;
Dan Gohmand2df6432010-04-09 02:00:38 +00004739
Bill Wendling86c5cbe2011-08-24 21:06:46 +00004740 // Ignore landingpad instructions.
David Majnemere09d0352016-03-24 21:40:22 +00004741 while (IP->isEHPad()) ++IP;
Bill Wendling86c5cbe2011-08-24 21:06:46 +00004742
Dan Gohmand2df6432010-04-09 02:00:38 +00004743 // Ignore debug intrinsics.
Dan Gohmand42e09d2010-03-26 00:33:27 +00004744 while (isa<DbgInfoIntrinsic>(IP)) ++IP;
Dan Gohman45774ce2010-02-12 10:34:29 +00004745
Andrew Trickc908b432012-01-20 07:41:13 +00004746 // Set IP below instructions recently inserted by SCEVExpander. This keeps the
4747 // IP consistent across expansions and allows the previously inserted
4748 // instructions to be reused by subsequent expansion.
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00004749 while (Rewriter.isInsertedInstruction(&*IP) && IP != LowestIP)
4750 ++IP;
Andrew Trickc908b432012-01-20 07:41:13 +00004751
Dan Gohmand2df6432010-04-09 02:00:38 +00004752 return IP;
4753}
4754
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004755/// Emit instructions for the leading candidate expression for this LSRUse (this
4756/// is called "expanding").
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00004757Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
4758 const Formula &F, BasicBlock::iterator IP,
Dan Gohmand2df6432010-04-09 02:00:38 +00004759 SCEVExpander &Rewriter,
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00004760 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
Andrew Trick57243da2013-10-25 21:35:56 +00004761 if (LU.RigidFormula)
4762 return LF.OperandValToReplace;
Dan Gohmand2df6432010-04-09 02:00:38 +00004763
4764 // Determine an input position which will be dominated by the operands and
4765 // which will dominate the result.
Andrew Trickc908b432012-01-20 07:41:13 +00004766 IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter);
Geoff Berryd0182802016-08-11 21:05:17 +00004767 Rewriter.setInsertPoint(&*IP);
Dan Gohmand2df6432010-04-09 02:00:38 +00004768
Dan Gohman45774ce2010-02-12 10:34:29 +00004769 // Inform the Rewriter if we have a post-increment use, so that it can
4770 // perform an advantageous expansion.
Dan Gohmand006ab92010-04-07 22:27:08 +00004771 Rewriter.setPostInc(LF.PostIncLoops);
Dan Gohman45774ce2010-02-12 10:34:29 +00004772
4773 // This is the type that the user actually needs.
Chris Lattner229907c2011-07-18 04:54:35 +00004774 Type *OpTy = LF.OperandValToReplace->getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00004775 // This will be the type that we'll initially expand to.
Chris Lattner229907c2011-07-18 04:54:35 +00004776 Type *Ty = F.getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00004777 if (!Ty)
4778 // No type known; just expand directly to the ultimate type.
4779 Ty = OpTy;
4780 else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy))
4781 // Expand directly to the ultimate type if it's the right size.
4782 Ty = OpTy;
4783 // This is the type to do integer arithmetic in.
Chris Lattner229907c2011-07-18 04:54:35 +00004784 Type *IntTy = SE.getEffectiveSCEVType(Ty);
Dan Gohman45774ce2010-02-12 10:34:29 +00004785
4786 // Build up a list of operands to add together to form the full base.
4787 SmallVector<const SCEV *, 8> Ops;
4788
4789 // Expand the BaseRegs portion.
Craig Topper77b99412015-05-23 08:01:41 +00004790 for (const SCEV *Reg : F.BaseRegs) {
Dan Gohman45774ce2010-02-12 10:34:29 +00004791 assert(!Reg->isZero() && "Zero allocated in a base register!");
4792
Dan Gohmand006ab92010-04-07 22:27:08 +00004793 // If we're expanding for a post-inc user, make the post-inc adjustment.
Sanjoy Dase3a15e82017-04-14 15:49:59 +00004794 Reg = denormalizeForPostIncUse(Reg, LF.PostIncLoops, SE);
Geoff Berryd0182802016-08-11 21:05:17 +00004795 Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, nullptr)));
Dan Gohman45774ce2010-02-12 10:34:29 +00004796 }
4797
4798 // Expand the ScaledReg portion.
Craig Topperf40110f2014-04-25 05:29:35 +00004799 Value *ICmpScaledV = nullptr;
Chandler Carruth6e479322013-01-07 15:04:40 +00004800 if (F.Scale != 0) {
Dan Gohman45774ce2010-02-12 10:34:29 +00004801 const SCEV *ScaledS = F.ScaledReg;
4802
Dan Gohmand006ab92010-04-07 22:27:08 +00004803 // If we're expanding for a post-inc user, make the post-inc adjustment.
4804 PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
Sanjoy Dase3a15e82017-04-14 15:49:59 +00004805 ScaledS = denormalizeForPostIncUse(ScaledS, Loops, SE);
Dan Gohman45774ce2010-02-12 10:34:29 +00004806
4807 if (LU.Kind == LSRUse::ICmpZero) {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00004808 // Expand ScaleReg as if it was part of the base regs.
4809 if (F.Scale == 1)
Sanjoy Das215df9e2015-08-04 01:52:05 +00004810 Ops.push_back(
Geoff Berryd0182802016-08-11 21:05:17 +00004811 SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr)));
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00004812 else {
4813 // An interesting way of "folding" with an icmp is to use a negated
4814 // scale, which we'll implement by inserting it into the other operand
4815 // of the icmp.
4816 assert(F.Scale == -1 &&
4817 "The only scale supported by ICmpZero uses is -1!");
Geoff Berryd0182802016-08-11 21:05:17 +00004818 ICmpScaledV = Rewriter.expandCodeFor(ScaledS, nullptr);
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00004819 }
Dan Gohman45774ce2010-02-12 10:34:29 +00004820 } else {
4821 // Otherwise just expand the scaled register and an explicit scale,
4822 // which is expected to be matched as part of the address.
Andrew Trick8370c7c2012-06-15 20:07:29 +00004823
4824 // Flush the operand list to suppress SCEVExpander hoisting address modes.
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00004825 // Unless the addressing mode will not be folded.
4826 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
4827 isAMCompletelyFolded(TTI, LU, F)) {
Geoff Berryd0182802016-08-11 21:05:17 +00004828 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty);
Andrew Trick8370c7c2012-06-15 20:07:29 +00004829 Ops.clear();
4830 Ops.push_back(SE.getUnknown(FullV));
4831 }
Geoff Berryd0182802016-08-11 21:05:17 +00004832 ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr));
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00004833 if (F.Scale != 1)
4834 ScaledS =
4835 SE.getMulExpr(ScaledS, SE.getConstant(ScaledS->getType(), F.Scale));
Dan Gohman45774ce2010-02-12 10:34:29 +00004836 Ops.push_back(ScaledS);
4837 }
4838 }
4839
Dan Gohman29707de2010-03-03 05:29:13 +00004840 // Expand the GV portion.
Chandler Carruth6e479322013-01-07 15:04:40 +00004841 if (F.BaseGV) {
Dan Gohman29707de2010-03-03 05:29:13 +00004842 // Flush the operand list to suppress SCEVExpander hoisting.
Andrew Trick8370c7c2012-06-15 20:07:29 +00004843 if (!Ops.empty()) {
Geoff Berryd0182802016-08-11 21:05:17 +00004844 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty);
Andrew Trick8370c7c2012-06-15 20:07:29 +00004845 Ops.clear();
4846 Ops.push_back(SE.getUnknown(FullV));
4847 }
Chandler Carruth6e479322013-01-07 15:04:40 +00004848 Ops.push_back(SE.getUnknown(F.BaseGV));
Andrew Trick8370c7c2012-06-15 20:07:29 +00004849 }
4850
4851 // Flush the operand list to suppress SCEVExpander hoisting of both folded and
4852 // unfolded offsets. LSR assumes they both live next to their uses.
4853 if (!Ops.empty()) {
Geoff Berryd0182802016-08-11 21:05:17 +00004854 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty);
Dan Gohman29707de2010-03-03 05:29:13 +00004855 Ops.clear();
4856 Ops.push_back(SE.getUnknown(FullV));
4857 }
4858
4859 // Expand the immediate portion.
Chandler Carruth6e479322013-01-07 15:04:40 +00004860 int64_t Offset = (uint64_t)F.BaseOffset + LF.Offset;
Dan Gohman45774ce2010-02-12 10:34:29 +00004861 if (Offset != 0) {
4862 if (LU.Kind == LSRUse::ICmpZero) {
4863 // The other interesting way of "folding" with an ICmpZero is to use a
4864 // negated immediate.
4865 if (!ICmpScaledV)
Eli Friedmanb46345d2011-10-13 23:48:33 +00004866 ICmpScaledV = ConstantInt::get(IntTy, -(uint64_t)Offset);
Dan Gohman45774ce2010-02-12 10:34:29 +00004867 else {
4868 Ops.push_back(SE.getUnknown(ICmpScaledV));
4869 ICmpScaledV = ConstantInt::get(IntTy, Offset);
4870 }
4871 } else {
4872 // Just add the immediate values. These again are expected to be matched
4873 // as part of the address.
Dan Gohman29707de2010-03-03 05:29:13 +00004874 Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset)));
Dan Gohman45774ce2010-02-12 10:34:29 +00004875 }
4876 }
4877
Dan Gohman6136e942011-05-03 00:46:49 +00004878 // Expand the unfolded offset portion.
4879 int64_t UnfoldedOffset = F.UnfoldedOffset;
4880 if (UnfoldedOffset != 0) {
4881 // Just add the immediate values.
4882 Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy,
4883 UnfoldedOffset)));
4884 }
4885
Dan Gohman45774ce2010-02-12 10:34:29 +00004886 // Emit instructions summing all the operands.
4887 const SCEV *FullS = Ops.empty() ?
Dan Gohman1d2ded72010-05-03 22:09:21 +00004888 SE.getConstant(IntTy, 0) :
Dan Gohman45774ce2010-02-12 10:34:29 +00004889 SE.getAddExpr(Ops);
Geoff Berryd0182802016-08-11 21:05:17 +00004890 Value *FullV = Rewriter.expandCodeFor(FullS, Ty);
Dan Gohman45774ce2010-02-12 10:34:29 +00004891
4892 // We're done expanding now, so reset the rewriter.
Dan Gohmand006ab92010-04-07 22:27:08 +00004893 Rewriter.clearPostInc();
Dan Gohman45774ce2010-02-12 10:34:29 +00004894
4895 // An ICmpZero Formula represents an ICmp which we're handling as a
4896 // comparison against zero. Now that we've expanded an expression for that
4897 // form, update the ICmp's other operand.
4898 if (LU.Kind == LSRUse::ICmpZero) {
4899 ICmpInst *CI = cast<ICmpInst>(LF.UserInst);
Benjamin Kramerf5e2fc42015-05-29 19:43:39 +00004900 DeadInsts.emplace_back(CI->getOperand(1));
Chandler Carruth6e479322013-01-07 15:04:40 +00004901 assert(!F.BaseGV && "ICmp does not support folding a global value and "
Dan Gohman45774ce2010-02-12 10:34:29 +00004902 "a scale at the same time!");
Chandler Carruth6e479322013-01-07 15:04:40 +00004903 if (F.Scale == -1) {
Dan Gohman45774ce2010-02-12 10:34:29 +00004904 if (ICmpScaledV->getType() != OpTy) {
4905 Instruction *Cast =
4906 CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false,
4907 OpTy, false),
4908 ICmpScaledV, OpTy, "tmp", CI);
4909 ICmpScaledV = Cast;
4910 }
4911 CI->setOperand(1, ICmpScaledV);
4912 } else {
Quentin Colombetc88baa5c2014-05-20 19:25:04 +00004913 // A scale of 1 means that the scale has been expanded as part of the
4914 // base regs.
4915 assert((F.Scale == 0 || F.Scale == 1) &&
Dan Gohman45774ce2010-02-12 10:34:29 +00004916 "ICmp does not support folding a global value and "
4917 "a scale at the same time!");
4918 Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy),
4919 -(uint64_t)Offset);
4920 if (C->getType() != OpTy)
4921 C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
4922 OpTy, false),
4923 C, OpTy);
4924
4925 CI->setOperand(1, C);
4926 }
4927 }
4928
4929 return FullV;
4930}
4931
Sanjoy Das94c4aec2015-08-16 18:22:46 +00004932/// Helper for Rewrite. PHI nodes are special because the use of their operands
4933/// effectively happens in their predecessor blocks, so the expression may need
4934/// to be expanded in multiple places.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00004935void LSRInstance::RewriteForPHI(
4936 PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F,
4937 SCEVExpander &Rewriter, SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
Dan Gohman6deab962010-02-16 20:25:07 +00004938 DenseMap<BasicBlock *, Value *> Inserted;
4939 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
4940 if (PN->getIncomingValue(i) == LF.OperandValToReplace) {
4941 BasicBlock *BB = PN->getIncomingBlock(i);
4942
4943 // If this is a critical edge, split the edge so that we do not insert
4944 // the code on all predecessor/successor paths. We do this unless this
4945 // is the canonical backedge for this loop, which complicates post-inc
4946 // users.
4947 if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 &&
David Majnemerbba17392017-01-13 22:24:27 +00004948 !isa<IndirectBrInst>(BB->getTerminator()) &&
4949 !isa<CatchSwitchInst>(BB->getTerminator())) {
Bill Wendling07efd6f2011-08-25 01:08:34 +00004950 BasicBlock *Parent = PN->getParent();
4951 Loop *PNLoop = LI.getLoopFor(Parent);
4952 if (!PNLoop || Parent != PNLoop->getHeader()) {
Dan Gohmande7f6992011-02-08 00:55:13 +00004953 // Split the critical edge.
Craig Topperf40110f2014-04-25 05:29:35 +00004954 BasicBlock *NewBB = nullptr;
Bill Wendling3fb137f2011-08-25 05:55:40 +00004955 if (!Parent->isLandingPad()) {
Chandler Carruth37df2cf2015-01-19 12:09:11 +00004956 NewBB = SplitCriticalEdge(BB, Parent,
4957 CriticalEdgeSplittingOptions(&DT, &LI)
4958 .setMergeIdenticalEdges()
4959 .setDontDeleteUselessPHIs());
Bill Wendling3fb137f2011-08-25 05:55:40 +00004960 } else {
4961 SmallVector<BasicBlock*, 2> NewBBs;
Chandler Carruth96ada252015-07-22 09:52:54 +00004962 SplitLandingPadPredecessors(Parent, BB, "", "", NewBBs, &DT, &LI);
Bill Wendling3fb137f2011-08-25 05:55:40 +00004963 NewBB = NewBBs[0];
4964 }
Andrew Trick402edbb2012-09-18 17:51:33 +00004965 // If NewBB==NULL, then SplitCriticalEdge refused to split because all
4966 // phi predecessors are identical. The simple thing to do is skip
4967 // splitting in this case rather than complicate the API.
4968 if (NewBB) {
4969 // If PN is outside of the loop and BB is in the loop, we want to
4970 // move the block to be immediately before the PHI block, not
4971 // immediately after BB.
4972 if (L->contains(BB) && !L->contains(PN))
4973 NewBB->moveBefore(PN->getParent());
Dan Gohman6deab962010-02-16 20:25:07 +00004974
Andrew Trick402edbb2012-09-18 17:51:33 +00004975 // Splitting the edge can reduce the number of PHI entries we have.
4976 e = PN->getNumIncomingValues();
4977 BB = NewBB;
4978 i = PN->getBasicBlockIndex(BB);
4979 }
Dan Gohmande7f6992011-02-08 00:55:13 +00004980 }
Dan Gohman6deab962010-02-16 20:25:07 +00004981 }
4982
4983 std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair =
Craig Topperf40110f2014-04-25 05:29:35 +00004984 Inserted.insert(std::make_pair(BB, static_cast<Value *>(nullptr)));
Dan Gohman6deab962010-02-16 20:25:07 +00004985 if (!Pair.second)
4986 PN->setIncomingValue(i, Pair.first->second);
4987 else {
Jonas Paulsson7a794222016-08-17 13:24:19 +00004988 Value *FullV = Expand(LU, LF, F, BB->getTerminator()->getIterator(),
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00004989 Rewriter, DeadInsts);
Dan Gohman6deab962010-02-16 20:25:07 +00004990
4991 // If this is reuse-by-noop-cast, insert the noop cast.
Chris Lattner229907c2011-07-18 04:54:35 +00004992 Type *OpTy = LF.OperandValToReplace->getType();
Dan Gohman6deab962010-02-16 20:25:07 +00004993 if (FullV->getType() != OpTy)
4994 FullV =
4995 CastInst::Create(CastInst::getCastOpcode(FullV, false,
4996 OpTy, false),
4997 FullV, LF.OperandValToReplace->getType(),
4998 "tmp", BB->getTerminator());
4999
5000 PN->setIncomingValue(i, FullV);
5001 Pair.first->second = FullV;
5002 }
5003 }
5004}
5005
Sanjoy Das94c4aec2015-08-16 18:22:46 +00005006/// Emit instructions for the leading candidate expression for this LSRUse (this
5007/// is called "expanding"), and update the UserInst to reference the newly
5008/// expanded value.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00005009void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
5010 const Formula &F, SCEVExpander &Rewriter,
5011 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
Dan Gohman45774ce2010-02-12 10:34:29 +00005012 // First, find an insertion point that dominates UserInst. For PHI nodes,
5013 // find the nearest block which dominates all the relevant uses.
5014 if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) {
Jonas Paulsson7a794222016-08-17 13:24:19 +00005015 RewriteForPHI(PN, LU, LF, F, Rewriter, DeadInsts);
Dan Gohman45774ce2010-02-12 10:34:29 +00005016 } else {
Duncan P. N. Exon Smithbe4d8cb2015-10-13 19:26:58 +00005017 Value *FullV =
Jonas Paulsson7a794222016-08-17 13:24:19 +00005018 Expand(LU, LF, F, LF.UserInst->getIterator(), Rewriter, DeadInsts);
Dan Gohman45774ce2010-02-12 10:34:29 +00005019
5020 // If this is reuse-by-noop-cast, insert the noop cast.
Chris Lattner229907c2011-07-18 04:54:35 +00005021 Type *OpTy = LF.OperandValToReplace->getType();
Dan Gohman45774ce2010-02-12 10:34:29 +00005022 if (FullV->getType() != OpTy) {
5023 Instruction *Cast =
5024 CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false),
5025 FullV, OpTy, "tmp", LF.UserInst);
5026 FullV = Cast;
5027 }
5028
5029 // Update the user. ICmpZero is handled specially here (for now) because
5030 // Expand may have updated one of the operands of the icmp already, and
5031 // its new value may happen to be equal to LF.OperandValToReplace, in
5032 // which case doing replaceUsesOfWith leads to replacing both operands
5033 // with the same value. TODO: Reorganize this.
Jonas Paulsson7a794222016-08-17 13:24:19 +00005034 if (LU.Kind == LSRUse::ICmpZero)
Dan Gohman45774ce2010-02-12 10:34:29 +00005035 LF.UserInst->setOperand(0, FullV);
5036 else
5037 LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV);
5038 }
5039
Benjamin Kramerf5e2fc42015-05-29 19:43:39 +00005040 DeadInsts.emplace_back(LF.OperandValToReplace);
Dan Gohman45774ce2010-02-12 10:34:29 +00005041}
5042
Sanjoy Das94c4aec2015-08-16 18:22:46 +00005043/// Rewrite all the fixup locations with new values, following the chosen
5044/// solution.
Justin Bogner843fb202015-12-15 19:40:57 +00005045void LSRInstance::ImplementSolution(
5046 const SmallVectorImpl<const Formula *> &Solution) {
Dan Gohman45774ce2010-02-12 10:34:29 +00005047 // Keep track of instructions we may have made dead, so that
5048 // we can remove them after we are done working.
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00005049 SmallVector<WeakTrackingVH, 16> DeadInsts;
Dan Gohman45774ce2010-02-12 10:34:29 +00005050
Mehdi Aminia28d91d2015-03-10 02:37:25 +00005051 SCEVExpander Rewriter(SE, L->getHeader()->getModule()->getDataLayout(),
5052 "lsr");
Andrew Trick4dc3eff2012-01-09 18:58:16 +00005053#ifndef NDEBUG
5054 Rewriter.setDebugType(DEBUG_TYPE);
5055#endif
Dan Gohman45774ce2010-02-12 10:34:29 +00005056 Rewriter.disableCanonicalMode();
Andrew Trick7fb669a2011-10-07 23:46:21 +00005057 Rewriter.enableLSRMode();
Dan Gohman45774ce2010-02-12 10:34:29 +00005058 Rewriter.setIVIncInsertPos(L, IVIncInsertPos);
5059
Andrew Trickd5d2db92012-01-10 01:45:08 +00005060 // Mark phi nodes that terminate chains so the expander tries to reuse them.
Craig Topper77b99412015-05-23 08:01:41 +00005061 for (const IVChain &Chain : IVChainVec) {
5062 if (PHINode *PN = dyn_cast<PHINode>(Chain.tailUserInst()))
Andrew Trickd5d2db92012-01-10 01:45:08 +00005063 Rewriter.setChainedPhi(PN);
5064 }
5065
Dan Gohman45774ce2010-02-12 10:34:29 +00005066 // Expand the new value definitions and update the users.
Jonas Paulsson7a794222016-08-17 13:24:19 +00005067 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
5068 for (const LSRFixup &Fixup : Uses[LUIdx].Fixups) {
5069 Rewrite(Uses[LUIdx], Fixup, *Solution[LUIdx], Rewriter, DeadInsts);
5070 Changed = true;
5071 }
Dan Gohman45774ce2010-02-12 10:34:29 +00005072
Craig Topper77b99412015-05-23 08:01:41 +00005073 for (const IVChain &Chain : IVChainVec) {
5074 GenerateIVChain(Chain, Rewriter, DeadInsts);
Andrew Trick248d4102012-01-09 21:18:52 +00005075 Changed = true;
5076 }
Dan Gohman45774ce2010-02-12 10:34:29 +00005077 // Clean up after ourselves. This must be done before deleting any
5078 // instructions.
5079 Rewriter.clear();
5080
5081 Changed |= DeleteTriviallyDeadInstructions(DeadInsts);
5082}
5083
Justin Bogner843fb202015-12-15 19:40:57 +00005084LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
5085 DominatorTree &DT, LoopInfo &LI,
5086 const TargetTransformInfo &TTI)
5087 : IU(IU), SE(SE), DT(DT), LI(LI), TTI(TTI), L(L), Changed(false),
5088 IVIncInsertPos(nullptr) {
Dan Gohmana83ac2d2009-11-05 21:11:53 +00005089 // If LoopSimplify form is not available, stay out of trouble.
Andrew Trick732ad802012-01-07 03:16:50 +00005090 if (!L->isLoopSimplifyForm())
5091 return;
Dan Gohmana83ac2d2009-11-05 21:11:53 +00005092
Andrew Trick070e5402012-03-16 03:16:56 +00005093 // If there's no interesting work to be done, bail early.
5094 if (IU.empty()) return;
5095
Andrew Trick19f80c12012-04-18 04:00:10 +00005096 // If there's too much analysis to be done, bail early. We won't be able to
5097 // model the problem anyway.
5098 unsigned NumUsers = 0;
Craig Topper77b99412015-05-23 08:01:41 +00005099 for (const IVStrideUse &U : IU) {
Andrew Trick19f80c12012-04-18 04:00:10 +00005100 if (++NumUsers > MaxIVUsers) {
Craig Topper37d0d862015-05-23 08:20:33 +00005101 (void)U;
Craig Topper77b99412015-05-23 08:01:41 +00005102 DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U << "\n");
Andrew Trick19f80c12012-04-18 04:00:10 +00005103 return;
5104 }
David Majnemera53b5bb2016-02-03 21:30:34 +00005105 // Bail out if we have a PHI on an EHPad that gets a value from a
5106 // CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is
5107 // no good place to stick any instructions.
5108 if (auto *PN = dyn_cast<PHINode>(U.getUser())) {
5109 auto *FirstNonPHI = PN->getParent()->getFirstNonPHI();
5110 if (isa<FuncletPadInst>(FirstNonPHI) ||
5111 isa<CatchSwitchInst>(FirstNonPHI))
5112 for (BasicBlock *PredBB : PN->blocks())
5113 if (isa<CatchSwitchInst>(PredBB->getFirstNonPHI()))
5114 return;
5115 }
Andrew Trick19f80c12012-04-18 04:00:10 +00005116 }
5117
Andrew Trick070e5402012-03-16 03:16:56 +00005118#ifndef NDEBUG
Andrew Trick12728f02012-01-17 06:45:52 +00005119 // All dominating loops must have preheaders, or SCEVExpander may not be able
5120 // to materialize an AddRecExpr whose Start is an outer AddRecExpr.
5121 //
Andrew Trick070e5402012-03-16 03:16:56 +00005122 // IVUsers analysis should only create users that are dominated by simple loop
5123 // headers. Since this loop should dominate all of its users, its user list
5124 // should be empty if this loop itself is not within a simple loop nest.
Andrew Trick12728f02012-01-17 06:45:52 +00005125 for (DomTreeNode *Rung = DT.getNode(L->getLoopPreheader());
5126 Rung; Rung = Rung->getIDom()) {
5127 BasicBlock *BB = Rung->getBlock();
5128 const Loop *DomLoop = LI.getLoopFor(BB);
5129 if (DomLoop && DomLoop->getHeader() == BB) {
Andrew Trick070e5402012-03-16 03:16:56 +00005130 assert(DomLoop->getLoopPreheader() && "LSR needs a simplified loop nest");
Andrew Trick12728f02012-01-17 06:45:52 +00005131 }
Andrew Trick732ad802012-01-07 03:16:50 +00005132 }
Andrew Trick070e5402012-03-16 03:16:56 +00005133#endif // DEBUG
Dan Gohman85875f72009-03-09 20:34:59 +00005134
Dan Gohman45774ce2010-02-12 10:34:29 +00005135 DEBUG(dbgs() << "\nLSR on loop ";
Chandler Carruthd48cdbf2014-01-09 02:29:41 +00005136 L->getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);
Dan Gohman45774ce2010-02-12 10:34:29 +00005137 dbgs() << ":\n");
Dan Gohmane201f8f2009-03-09 20:46:50 +00005138
Dan Gohman927bcaa2010-05-20 20:33:18 +00005139 // First, perform some low-level loop optimizations.
Dan Gohman45774ce2010-02-12 10:34:29 +00005140 OptimizeShadowIV();
Dan Gohman4c4043c2010-05-20 20:05:31 +00005141 OptimizeLoopTermCond();
Evan Cheng78a4eb82009-05-11 22:33:01 +00005142
Andrew Trick8acb4342011-07-21 00:40:04 +00005143 // If loop preparation eliminates all interesting IV users, bail.
5144 if (IU.empty()) return;
5145
Andrew Trick168dfff2011-09-29 01:53:08 +00005146 // Skip nested loops until we can model them better with formulae.
Andrew Trickd97b83e2012-03-22 22:42:45 +00005147 if (!L->empty()) {
Andrew Trickbc6de902011-09-29 01:33:38 +00005148 DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
Andrew Trick168dfff2011-09-29 01:53:08 +00005149 return;
Andrew Trickbc6de902011-09-29 01:33:38 +00005150 }
5151
Dan Gohman927bcaa2010-05-20 20:33:18 +00005152 // Start collecting data and preparing for the solver.
Andrew Trick29fe5f02012-01-09 19:50:34 +00005153 CollectChains();
Dan Gohman45774ce2010-02-12 10:34:29 +00005154 CollectInterestingTypesAndFactors();
5155 CollectFixupsAndInitialFormulae();
5156 CollectLoopInvariantFixupsAndFormulae();
Chris Lattner9bfa6f82005-08-08 05:28:22 +00005157
Andrew Trick248d4102012-01-09 21:18:52 +00005158 assert(!Uses.empty() && "IVUsers reported at least one use");
Dan Gohman45774ce2010-02-12 10:34:29 +00005159 DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
5160 print_uses(dbgs()));
Misha Brukmanb1c93172005-04-21 23:48:37 +00005161
Dan Gohman45774ce2010-02-12 10:34:29 +00005162 // Now use the reuse data to generate a bunch of interesting ways
5163 // to formulate the values needed for the uses.
5164 GenerateAllReuseFormulae();
Evan Cheng3df447d2006-03-16 21:53:05 +00005165
Dan Gohman45774ce2010-02-12 10:34:29 +00005166 FilterOutUndesirableDedicatedRegisters();
5167 NarrowSearchSpaceUsingHeuristics();
Dan Gohman92c36962009-12-18 00:06:20 +00005168
Dan Gohman45774ce2010-02-12 10:34:29 +00005169 SmallVector<const Formula *, 8> Solution;
5170 Solve(Solution);
Dan Gohman92c36962009-12-18 00:06:20 +00005171
Dan Gohman45774ce2010-02-12 10:34:29 +00005172 // Release memory that is no longer needed.
5173 Factors.clear();
5174 Types.clear();
5175 RegUses.clear();
5176
Andrew Trick58124392011-09-27 00:44:14 +00005177 if (Solution.empty())
5178 return;
5179
Dan Gohman45774ce2010-02-12 10:34:29 +00005180#ifndef NDEBUG
5181 // Formulae should be legal.
Craig Topper77b99412015-05-23 08:01:41 +00005182 for (const LSRUse &LU : Uses) {
5183 for (const Formula &F : LU.Formulae)
Chandler Carruth26c59fa2013-01-07 14:41:08 +00005184 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
Craig Topper77b99412015-05-23 08:01:41 +00005185 F) && "Illegal formula generated!");
Dan Gohman45774ce2010-02-12 10:34:29 +00005186 };
5187#endif
5188
5189 // Now that we've decided what we want, make it so.
Justin Bogner843fb202015-12-15 19:40:57 +00005190 ImplementSolution(Solution);
Dan Gohman45774ce2010-02-12 10:34:29 +00005191}
5192
5193void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
5194 if (Factors.empty() && Types.empty()) return;
5195
5196 OS << "LSR has identified the following interesting factors and types: ";
5197 bool First = true;
5198
Craig Topper10949ae2015-05-23 08:45:10 +00005199 for (int64_t Factor : Factors) {
Dan Gohman45774ce2010-02-12 10:34:29 +00005200 if (!First) OS << ", ";
5201 First = false;
Craig Topper10949ae2015-05-23 08:45:10 +00005202 OS << '*' << Factor;
Evan Cheng87fe40b2009-11-10 21:14:05 +00005203 }
Dale Johannesen02cb2bf2009-05-11 17:15:42 +00005204
Craig Topper10949ae2015-05-23 08:45:10 +00005205 for (Type *Ty : Types) {
Dan Gohman45774ce2010-02-12 10:34:29 +00005206 if (!First) OS << ", ";
5207 First = false;
Craig Topper10949ae2015-05-23 08:45:10 +00005208 OS << '(' << *Ty << ')';
Dan Gohman45774ce2010-02-12 10:34:29 +00005209 }
5210 OS << '\n';
5211}
5212
5213void LSRInstance::print_fixups(raw_ostream &OS) const {
5214 OS << "LSR is examining the following fixup sites:\n";
Jonas Paulsson7a794222016-08-17 13:24:19 +00005215 for (const LSRUse &LU : Uses)
5216 for (const LSRFixup &LF : LU.Fixups) {
5217 dbgs() << " ";
5218 LF.print(OS);
5219 OS << '\n';
5220 }
Dan Gohman45774ce2010-02-12 10:34:29 +00005221}
5222
5223void LSRInstance::print_uses(raw_ostream &OS) const {
5224 OS << "LSR is examining the following uses:\n";
Craig Topper77b99412015-05-23 08:01:41 +00005225 for (const LSRUse &LU : Uses) {
Dan Gohman45774ce2010-02-12 10:34:29 +00005226 dbgs() << " ";
5227 LU.print(OS);
5228 OS << '\n';
Craig Topper77b99412015-05-23 08:01:41 +00005229 for (const Formula &F : LU.Formulae) {
Dan Gohman45774ce2010-02-12 10:34:29 +00005230 OS << " ";
Craig Topper77b99412015-05-23 08:01:41 +00005231 F.print(OS);
Dan Gohman45774ce2010-02-12 10:34:29 +00005232 OS << '\n';
5233 }
5234 }
5235}
5236
5237void LSRInstance::print(raw_ostream &OS) const {
5238 print_factors_and_types(OS);
5239 print_fixups(OS);
5240 print_uses(OS);
5241}
5242
Matthias Braun8c209aa2017-01-28 02:02:38 +00005243#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5244LLVM_DUMP_METHOD void LSRInstance::dump() const {
Dan Gohman45774ce2010-02-12 10:34:29 +00005245 print(errs()); errs() << '\n';
5246}
Matthias Braun8c209aa2017-01-28 02:02:38 +00005247#endif
Dan Gohman45774ce2010-02-12 10:34:29 +00005248
5249namespace {
5250
5251class LoopStrengthReduce : public LoopPass {
Dan Gohman45774ce2010-02-12 10:34:29 +00005252public:
5253 static char ID; // Pass ID, replacement for typeid
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00005254
Chandler Carruth26c59fa2013-01-07 14:41:08 +00005255 LoopStrengthReduce();
Dan Gohman45774ce2010-02-12 10:34:29 +00005256
5257private:
Craig Topper3e4c6972014-03-05 09:10:37 +00005258 bool runOnLoop(Loop *L, LPPassManager &LPM) override;
5259 void getAnalysisUsage(AnalysisUsage &AU) const override;
Dan Gohman45774ce2010-02-12 10:34:29 +00005260};
Dan Gohman45774ce2010-02-12 10:34:29 +00005261
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00005262} // end anonymous namespace
Dan Gohman45774ce2010-02-12 10:34:29 +00005263
Chandler Carruth26c59fa2013-01-07 14:41:08 +00005264LoopStrengthReduce::LoopStrengthReduce() : LoopPass(ID) {
5265 initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry());
5266}
Dan Gohman45774ce2010-02-12 10:34:29 +00005267
5268void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
5269 // We split critical edges, so we change the CFG. However, we do update
5270 // many analyses if they are around.
Eric Christopherda6bd452011-02-10 01:48:24 +00005271 AU.addPreservedID(LoopSimplifyID);
Dan Gohman45774ce2010-02-12 10:34:29 +00005272
Chandler Carruth4f8f3072015-01-17 14:16:18 +00005273 AU.addRequired<LoopInfoWrapperPass>();
5274 AU.addPreserved<LoopInfoWrapperPass>();
Eric Christopherda6bd452011-02-10 01:48:24 +00005275 AU.addRequiredID(LoopSimplifyID);
Chandler Carruth73523022014-01-13 13:07:17 +00005276 AU.addRequired<DominatorTreeWrapperPass>();
5277 AU.addPreserved<DominatorTreeWrapperPass>();
Chandler Carruth2f1fd162015-08-17 02:08:17 +00005278 AU.addRequired<ScalarEvolutionWrapperPass>();
5279 AU.addPreserved<ScalarEvolutionWrapperPass>();
Cameron Zwarich97dae4d2011-02-10 23:53:14 +00005280 // Requiring LoopSimplify a second time here prevents IVUsers from running
5281 // twice, since LoopSimplify was invalidated by running ScalarEvolution.
5282 AU.addRequiredID(LoopSimplifyID);
Dehao Chen1a444522016-07-16 22:51:33 +00005283 AU.addRequired<IVUsersWrapperPass>();
5284 AU.addPreserved<IVUsersWrapperPass>();
Chandler Carruth705b1852015-01-31 03:43:40 +00005285 AU.addRequired<TargetTransformInfoWrapperPass>();
Dan Gohman45774ce2010-02-12 10:34:29 +00005286}
5287
Dehao Chen6132ee82016-07-18 21:41:50 +00005288static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
5289 DominatorTree &DT, LoopInfo &LI,
5290 const TargetTransformInfo &TTI) {
Dan Gohman45774ce2010-02-12 10:34:29 +00005291 bool Changed = false;
5292
5293 // Run the main LSR transformation.
Justin Bogner843fb202015-12-15 19:40:57 +00005294 Changed |= LSRInstance(L, IU, SE, DT, LI, TTI).getChanged();
Dan Gohman45774ce2010-02-12 10:34:29 +00005295
Andrew Trick2ec61a82012-01-07 01:36:44 +00005296 // Remove any extra phis created by processing inner loops.
Dan Gohmanb5358002010-01-05 16:31:45 +00005297 Changed |= DeleteDeadPHIs(L->getHeader());
Andrew Trickf950ce82013-01-06 05:59:39 +00005298 if (EnablePhiElim && L->isLoopSimplifyForm()) {
Sanjoy Dase6bca0e2017-05-01 17:07:49 +00005299 SmallVector<WeakTrackingVH, 16> DeadInsts;
Mehdi Aminia28d91d2015-03-10 02:37:25 +00005300 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
Dehao Chen6132ee82016-07-18 21:41:50 +00005301 SCEVExpander Rewriter(SE, DL, "lsr");
Andrew Trick2ec61a82012-01-07 01:36:44 +00005302#ifndef NDEBUG
5303 Rewriter.setDebugType(DEBUG_TYPE);
5304#endif
Dehao Chen6132ee82016-07-18 21:41:50 +00005305 unsigned numFolded = Rewriter.replaceCongruentIVs(L, &DT, DeadInsts, &TTI);
Andrew Trick2ec61a82012-01-07 01:36:44 +00005306 if (numFolded) {
5307 Changed = true;
5308 DeleteTriviallyDeadInstructions(DeadInsts);
5309 DeleteDeadPHIs(L->getHeader());
5310 }
5311 }
Evan Cheng03001cb2008-07-07 19:51:32 +00005312 return Changed;
Nate Begemanb18121e2004-10-18 21:08:22 +00005313}
Dehao Chen6132ee82016-07-18 21:41:50 +00005314
5315bool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
5316 if (skipLoop(L))
5317 return false;
5318
5319 auto &IU = getAnalysis<IVUsersWrapperPass>().getIU();
5320 auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
5321 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
5322 auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
5323 const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
5324 *L->getHeader()->getParent());
5325 return ReduceLoopStrength(L, IU, SE, DT, LI, TTI);
5326}
5327
Chandler Carruth410eaeb2017-01-11 06:23:21 +00005328PreservedAnalyses LoopStrengthReducePass::run(Loop &L, LoopAnalysisManager &AM,
5329 LoopStandardAnalysisResults &AR,
5330 LPMUpdater &) {
5331 if (!ReduceLoopStrength(&L, AM.getResult<IVUsersAnalysis>(L, AR), AR.SE,
5332 AR.DT, AR.LI, AR.TTI))
Dehao Chen6132ee82016-07-18 21:41:50 +00005333 return PreservedAnalyses::all();
5334
5335 return getLoopPassPreservedAnalyses();
5336}
Eugene Zelenkoa3fe70d2016-11-30 17:48:10 +00005337
5338char LoopStrengthReduce::ID = 0;
5339INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",
5340 "Loop Strength Reduction", false, false)
5341INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
5342INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
5343INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
5344INITIALIZE_PASS_DEPENDENCY(IVUsersWrapperPass)
5345INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
5346INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
5347INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",
5348 "Loop Strength Reduction", false, false)
5349
5350Pass *llvm::createLoopStrengthReducePass() { return new LoopStrengthReduce(); }