blob: 714065234f88ba7185ae11138ac037c3fa390b71 [file] [log] [blame]
Chris Lattner53e677a2004-04-02 20:23:17 +00001//===- ScalarEvolution.cpp - Scalar Evolution Analysis ----------*- C++ -*-===//
Misha Brukman2b37d7c2005-04-21 21:13:18 +00002//
Chris Lattner53e677a2004-04-02 20:23:17 +00003// The LLVM Compiler Infrastructure
4//
Chris Lattner4ee451d2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Misha Brukman2b37d7c2005-04-21 21:13:18 +00007//
Chris Lattner53e677a2004-04-02 20:23:17 +00008//===----------------------------------------------------------------------===//
9//
10// This file contains the implementation of the scalar evolution analysis
11// engine, which is used primarily to analyze expressions involving induction
12// variables in loops.
13//
14// There are several aspects to this library. First is the representation of
15// scalar expressions, which are represented as subclasses of the SCEV class.
16// These classes are used to represent certain types of subexpressions that we
Dan Gohmanbc3d77a2009-07-25 16:18:07 +000017// can handle. We only create one SCEV of a particular shape, so
18// pointer-comparisons for equality are legal.
Chris Lattner53e677a2004-04-02 20:23:17 +000019//
20// One important aspect of the SCEV objects is that they are never cyclic, even
21// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
22// the PHI node is one of the idioms that we can represent (e.g., a polynomial
23// recurrence) then we represent it directly as a recurrence node, otherwise we
24// represent it as a SCEVUnknown node.
25//
26// In addition to being able to represent expressions of various types, we also
27// have folders that are used to build the *canonical* representation for a
28// particular expression. These folders are capable of using a variety of
29// rewrite rules to simplify the expressions.
Misha Brukman2b37d7c2005-04-21 21:13:18 +000030//
Chris Lattner53e677a2004-04-02 20:23:17 +000031// Once the folders are defined, we can implement the more interesting
32// higher-level code, such as the code that recognizes PHI nodes of various
33// types, computes the execution count of a loop, etc.
34//
Chris Lattner53e677a2004-04-02 20:23:17 +000035// TODO: We should use these routines and value representations to implement
36// dependence analysis!
37//
38//===----------------------------------------------------------------------===//
39//
40// There are several good references for the techniques used in this analysis.
41//
42// Chains of recurrences -- a method to expedite the evaluation
43// of closed-form functions
44// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
45//
46// On computational properties of chains of recurrences
47// Eugene V. Zima
48//
49// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
50// Robert A. van Engelen
51//
52// Efficient Symbolic Analysis for Optimizing Compilers
53// Robert A. van Engelen
54//
55// Using the chains of recurrences algebra for data dependence testing and
56// induction variable substitution
57// MS Thesis, Johnie Birch
58//
59//===----------------------------------------------------------------------===//
60
Chris Lattner3b27d682006-12-19 22:30:33 +000061#define DEBUG_TYPE "scalar-evolution"
Chris Lattner0a7f98c2004-04-15 15:07:24 +000062#include "llvm/Analysis/ScalarEvolutionExpressions.h"
Chris Lattner53e677a2004-04-02 20:23:17 +000063#include "llvm/Constants.h"
64#include "llvm/DerivedTypes.h"
Chris Lattner673e02b2004-10-12 01:49:27 +000065#include "llvm/GlobalVariable.h"
Dan Gohman26812322009-08-25 17:49:57 +000066#include "llvm/GlobalAlias.h"
Chris Lattner53e677a2004-04-02 20:23:17 +000067#include "llvm/Instructions.h"
Owen Anderson76f600b2009-07-06 22:37:39 +000068#include "llvm/LLVMContext.h"
Dan Gohmanca178902009-07-17 20:47:02 +000069#include "llvm/Operator.h"
John Criswella1156432005-10-27 15:54:34 +000070#include "llvm/Analysis/ConstantFolding.h"
Evan Cheng5a6c1a82009-02-17 00:13:06 +000071#include "llvm/Analysis/Dominators.h"
Chris Lattner53e677a2004-04-02 20:23:17 +000072#include "llvm/Analysis/LoopInfo.h"
Dan Gohman61ffa8e2009-06-16 19:52:01 +000073#include "llvm/Analysis/ValueTracking.h"
Chris Lattner53e677a2004-04-02 20:23:17 +000074#include "llvm/Assembly/Writer.h"
Dan Gohman2d1be872009-04-16 03:18:22 +000075#include "llvm/Target/TargetData.h"
Chris Lattner95255282006-06-28 23:17:24 +000076#include "llvm/Support/CommandLine.h"
Chris Lattner53e677a2004-04-02 20:23:17 +000077#include "llvm/Support/ConstantRange.h"
David Greene63c94632009-12-23 22:58:38 +000078#include "llvm/Support/Debug.h"
Torok Edwinc25e7582009-07-11 20:10:48 +000079#include "llvm/Support/ErrorHandling.h"
Dan Gohman2d1be872009-04-16 03:18:22 +000080#include "llvm/Support/GetElementPtrTypeIterator.h"
Chris Lattner53e677a2004-04-02 20:23:17 +000081#include "llvm/Support/InstIterator.h"
Chris Lattner75de5ab2006-12-19 01:16:02 +000082#include "llvm/Support/MathExtras.h"
Dan Gohmanb7ef7292009-04-21 00:47:46 +000083#include "llvm/Support/raw_ostream.h"
Reid Spencer551ccae2004-09-01 22:55:40 +000084#include "llvm/ADT/Statistic.h"
Dan Gohman2d1be872009-04-16 03:18:22 +000085#include "llvm/ADT/STLExtras.h"
Dan Gohman59ae6b92009-07-08 19:23:34 +000086#include "llvm/ADT/SmallPtrSet.h"
Alkis Evlogimenos20aa4742004-09-03 18:19:51 +000087#include <algorithm>
Chris Lattner53e677a2004-04-02 20:23:17 +000088using namespace llvm;
89
Chris Lattner3b27d682006-12-19 22:30:33 +000090STATISTIC(NumArrayLenItCounts,
91 "Number of trip counts computed with array length");
92STATISTIC(NumTripCountsComputed,
93 "Number of loops with predictable loop counts");
94STATISTIC(NumTripCountsNotComputed,
95 "Number of loops without predictable loop counts");
96STATISTIC(NumBruteForceTripCountsComputed,
97 "Number of loops with trip counts computed by force");
98
Dan Gohman844731a2008-05-13 00:00:25 +000099static cl::opt<unsigned>
Chris Lattner3b27d682006-12-19 22:30:33 +0000100MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
101 cl::desc("Maximum number of iterations SCEV will "
Dan Gohman64a845e2009-06-24 04:48:43 +0000102 "symbolically execute a constant "
103 "derived loop"),
Chris Lattner3b27d682006-12-19 22:30:33 +0000104 cl::init(100));
105
Owen Andersond13db2c2010-07-21 22:09:45 +0000106INITIALIZE_PASS(ScalarEvolution, "scalar-evolution",
107 "Scalar Evolution Analysis", false, true);
Devang Patel19974732007-05-03 01:11:54 +0000108char ScalarEvolution::ID = 0;
Chris Lattner53e677a2004-04-02 20:23:17 +0000109
110//===----------------------------------------------------------------------===//
111// SCEV class definitions
112//===----------------------------------------------------------------------===//
113
114//===----------------------------------------------------------------------===//
115// Implementation of the SCEV class.
116//
Dan Gohmanc39f44b2009-06-30 20:13:32 +0000117
Chris Lattner53e677a2004-04-02 20:23:17 +0000118SCEV::~SCEV() {}
Dan Gohmanc39f44b2009-06-30 20:13:32 +0000119
Chris Lattner53e677a2004-04-02 20:23:17 +0000120void SCEV::dump() const {
David Greene25e0e872009-12-23 22:18:14 +0000121 print(dbgs());
122 dbgs() << '\n';
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000123}
124
Dan Gohmancfeb6a42008-06-18 16:23:07 +0000125bool SCEV::isZero() const {
126 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
127 return SC->getValue()->isZero();
128 return false;
129}
130
Dan Gohman70a1fe72009-05-18 15:22:39 +0000131bool SCEV::isOne() const {
132 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
133 return SC->getValue()->isOne();
134 return false;
135}
Chris Lattner53e677a2004-04-02 20:23:17 +0000136
Dan Gohman4d289bf2009-06-24 00:30:26 +0000137bool SCEV::isAllOnesValue() const {
138 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
139 return SC->getValue()->isAllOnesValue();
140 return false;
141}
142
Owen Anderson753ad612009-06-22 21:57:23 +0000143SCEVCouldNotCompute::SCEVCouldNotCompute() :
Dan Gohman3bf63762010-06-18 19:54:20 +0000144 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {}
Dan Gohman1c343752009-06-27 21:21:31 +0000145
Chris Lattner53e677a2004-04-02 20:23:17 +0000146bool SCEVCouldNotCompute::isLoopInvariant(const Loop *L) const {
Torok Edwinc23197a2009-07-14 16:55:14 +0000147 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
Misha Brukmanbb2aff12004-04-05 19:00:46 +0000148 return false;
Chris Lattner53e677a2004-04-02 20:23:17 +0000149}
150
151const Type *SCEVCouldNotCompute::getType() const {
Torok Edwinc23197a2009-07-14 16:55:14 +0000152 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
Misha Brukmanbb2aff12004-04-05 19:00:46 +0000153 return 0;
Chris Lattner53e677a2004-04-02 20:23:17 +0000154}
155
156bool SCEVCouldNotCompute::hasComputableLoopEvolution(const Loop *L) const {
Torok Edwinc23197a2009-07-14 16:55:14 +0000157 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
Chris Lattner53e677a2004-04-02 20:23:17 +0000158 return false;
159}
160
Dan Gohmanfef8bb22009-07-25 01:13:03 +0000161bool SCEVCouldNotCompute::hasOperand(const SCEV *) const {
162 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
163 return false;
Chris Lattner4dc534c2005-02-13 04:37:18 +0000164}
165
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000166void SCEVCouldNotCompute::print(raw_ostream &OS) const {
Chris Lattner53e677a2004-04-02 20:23:17 +0000167 OS << "***COULDNOTCOMPUTE***";
168}
169
170bool SCEVCouldNotCompute::classof(const SCEV *S) {
171 return S->getSCEVType() == scCouldNotCompute;
172}
173
Dan Gohman0bba49c2009-07-07 17:06:11 +0000174const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
Dan Gohman1c343752009-06-27 21:21:31 +0000175 FoldingSetNodeID ID;
176 ID.AddInteger(scConstant);
177 ID.AddPointer(V);
178 void *IP = 0;
179 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
Dan Gohman3bf63762010-06-18 19:54:20 +0000180 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
Dan Gohman1c343752009-06-27 21:21:31 +0000181 UniqueSCEVs.InsertNode(S, IP);
182 return S;
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000183}
Chris Lattner53e677a2004-04-02 20:23:17 +0000184
Dan Gohman0bba49c2009-07-07 17:06:11 +0000185const SCEV *ScalarEvolution::getConstant(const APInt& Val) {
Owen Andersoneed707b2009-07-24 23:12:02 +0000186 return getConstant(ConstantInt::get(getContext(), Val));
Dan Gohman9a6ae962007-07-09 15:25:17 +0000187}
188
Dan Gohman0bba49c2009-07-07 17:06:11 +0000189const SCEV *
Dan Gohman6de29f82009-06-15 22:12:54 +0000190ScalarEvolution::getConstant(const Type *Ty, uint64_t V, bool isSigned) {
Dan Gohmana560fd22010-04-21 16:04:04 +0000191 const IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
192 return getConstant(ConstantInt::get(ITy, V, isSigned));
Dan Gohman6de29f82009-06-15 22:12:54 +0000193}
194
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000195const Type *SCEVConstant::getType() const { return V->getType(); }
Chris Lattner53e677a2004-04-02 20:23:17 +0000196
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000197void SCEVConstant::print(raw_ostream &OS) const {
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000198 WriteAsOperand(OS, V, false);
199}
Chris Lattner53e677a2004-04-02 20:23:17 +0000200
Dan Gohman3bf63762010-06-18 19:54:20 +0000201SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID,
Dan Gohmanc050fd92009-07-13 20:50:19 +0000202 unsigned SCEVTy, const SCEV *op, const Type *ty)
Dan Gohman3bf63762010-06-18 19:54:20 +0000203 : SCEV(ID, SCEVTy), Op(op), Ty(ty) {}
Dan Gohman1c343752009-06-27 21:21:31 +0000204
Dan Gohman84923602009-04-21 01:25:57 +0000205bool SCEVCastExpr::dominates(BasicBlock *BB, DominatorTree *DT) const {
206 return Op->dominates(BB, DT);
207}
208
Dan Gohman6e70e312009-09-27 15:26:03 +0000209bool SCEVCastExpr::properlyDominates(BasicBlock *BB, DominatorTree *DT) const {
210 return Op->properlyDominates(BB, DT);
211}
212
Dan Gohman3bf63762010-06-18 19:54:20 +0000213SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
Dan Gohmanc050fd92009-07-13 20:50:19 +0000214 const SCEV *op, const Type *ty)
Dan Gohman3bf63762010-06-18 19:54:20 +0000215 : SCEVCastExpr(ID, scTruncate, op, ty) {
Duncan Sands1df98592010-02-16 11:11:14 +0000216 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
217 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000218 "Cannot truncate non-integer value!");
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000219}
Chris Lattner53e677a2004-04-02 20:23:17 +0000220
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000221void SCEVTruncateExpr::print(raw_ostream &OS) const {
Dan Gohman36b8e532009-04-29 20:27:52 +0000222 OS << "(trunc " << *Op->getType() << " " << *Op << " to " << *Ty << ")";
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000223}
224
Dan Gohman3bf63762010-06-18 19:54:20 +0000225SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
Dan Gohmanc050fd92009-07-13 20:50:19 +0000226 const SCEV *op, const Type *ty)
Dan Gohman3bf63762010-06-18 19:54:20 +0000227 : SCEVCastExpr(ID, scZeroExtend, op, ty) {
Duncan Sands1df98592010-02-16 11:11:14 +0000228 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
229 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000230 "Cannot zero extend non-integer value!");
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000231}
232
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000233void SCEVZeroExtendExpr::print(raw_ostream &OS) const {
Dan Gohman36b8e532009-04-29 20:27:52 +0000234 OS << "(zext " << *Op->getType() << " " << *Op << " to " << *Ty << ")";
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000235}
236
Dan Gohman3bf63762010-06-18 19:54:20 +0000237SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
Dan Gohmanc050fd92009-07-13 20:50:19 +0000238 const SCEV *op, const Type *ty)
Dan Gohman3bf63762010-06-18 19:54:20 +0000239 : SCEVCastExpr(ID, scSignExtend, op, ty) {
Duncan Sands1df98592010-02-16 11:11:14 +0000240 assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
241 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Dan Gohmand19534a2007-06-15 14:38:12 +0000242 "Cannot sign extend non-integer value!");
Dan Gohmand19534a2007-06-15 14:38:12 +0000243}
244
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000245void SCEVSignExtendExpr::print(raw_ostream &OS) const {
Dan Gohman36b8e532009-04-29 20:27:52 +0000246 OS << "(sext " << *Op->getType() << " " << *Op << " to " << *Ty << ")";
Dan Gohmand19534a2007-06-15 14:38:12 +0000247}
248
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000249void SCEVCommutativeExpr::print(raw_ostream &OS) const {
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000250 const char *OpStr = getOperationStr();
Dan Gohmana5145c82010-04-16 15:03:25 +0000251 OS << "(";
252 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I) {
253 OS << **I;
Oscar Fuentesee56c422010-08-02 06:00:15 +0000254 if (llvm::next(I) != E)
Dan Gohmana5145c82010-04-16 15:03:25 +0000255 OS << OpStr;
256 }
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000257 OS << ")";
258}
259
Dan Gohmanecb403a2009-05-07 14:00:19 +0000260bool SCEVNAryExpr::dominates(BasicBlock *BB, DominatorTree *DT) const {
Dan Gohmanbb854092010-08-16 16:16:11 +0000261 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I)
262 if (!(*I)->dominates(BB, DT))
Evan Cheng5a6c1a82009-02-17 00:13:06 +0000263 return false;
Evan Cheng5a6c1a82009-02-17 00:13:06 +0000264 return true;
265}
266
Dan Gohman6e70e312009-09-27 15:26:03 +0000267bool SCEVNAryExpr::properlyDominates(BasicBlock *BB, DominatorTree *DT) const {
Dan Gohmanbb854092010-08-16 16:16:11 +0000268 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I)
269 if (!(*I)->properlyDominates(BB, DT))
Dan Gohman6e70e312009-09-27 15:26:03 +0000270 return false;
Dan Gohman6e70e312009-09-27 15:26:03 +0000271 return true;
272}
273
Dan Gohman2f199f92010-08-16 16:21:27 +0000274bool SCEVNAryExpr::isLoopInvariant(const Loop *L) const {
275 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I)
276 if (!(*I)->isLoopInvariant(L))
277 return false;
278 return true;
279}
280
281// hasComputableLoopEvolution - N-ary expressions have computable loop
282// evolutions iff they have at least one operand that varies with the loop,
283// but that all varying operands are computable.
284bool SCEVNAryExpr::hasComputableLoopEvolution(const Loop *L) const {
285 bool HasVarying = false;
286 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I) {
287 const SCEV *S = *I;
288 if (!S->isLoopInvariant(L)) {
289 if (S->hasComputableLoopEvolution(L))
290 HasVarying = true;
291 else
292 return false;
293 }
294 }
295 return HasVarying;
296}
297
298bool SCEVNAryExpr::hasOperand(const SCEV *O) const {
299 for (op_iterator I = op_begin(), E = op_end(); I != E; ++I) {
300 const SCEV *S = *I;
301 if (O == S || S->hasOperand(O))
302 return true;
303 }
304 return false;
305}
306
Evan Cheng5a6c1a82009-02-17 00:13:06 +0000307bool SCEVUDivExpr::dominates(BasicBlock *BB, DominatorTree *DT) const {
308 return LHS->dominates(BB, DT) && RHS->dominates(BB, DT);
309}
310
Dan Gohman6e70e312009-09-27 15:26:03 +0000311bool SCEVUDivExpr::properlyDominates(BasicBlock *BB, DominatorTree *DT) const {
312 return LHS->properlyDominates(BB, DT) && RHS->properlyDominates(BB, DT);
313}
314
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000315void SCEVUDivExpr::print(raw_ostream &OS) const {
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000316 OS << "(" << *LHS << " /u " << *RHS << ")";
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000317}
318
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000319const Type *SCEVUDivExpr::getType() const {
Dan Gohman91bb61a2009-05-26 17:44:05 +0000320 // In most cases the types of LHS and RHS will be the same, but in some
321 // crazy cases one or the other may be a pointer. ScalarEvolution doesn't
322 // depend on the type for correctness, but handling types carefully can
323 // avoid extra casts in the SCEVExpander. The LHS is more likely to be
324 // a pointer type than the RHS, so use the RHS' type here.
325 return RHS->getType();
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000326}
327
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000328bool SCEVAddRecExpr::isLoopInvariant(const Loop *QueryLoop) const {
Dan Gohmana3035a62009-05-20 01:01:24 +0000329 // Add recurrences are never invariant in the function-body (null loop).
Dan Gohmane890eea2009-06-26 22:17:21 +0000330 if (!QueryLoop)
331 return false;
332
333 // This recurrence is variant w.r.t. QueryLoop if QueryLoop contains L.
Dan Gohman92329c72009-12-18 01:24:09 +0000334 if (QueryLoop->contains(L))
Dan Gohmane890eea2009-06-26 22:17:21 +0000335 return false;
336
Dan Gohman71c41442010-08-13 20:11:39 +0000337 // This recurrence is invariant w.r.t. QueryLoop if L contains QueryLoop.
338 if (L->contains(QueryLoop))
339 return true;
340
Dan Gohmane890eea2009-06-26 22:17:21 +0000341 // This recurrence is variant w.r.t. QueryLoop if any of its operands
342 // are variant.
343 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
344 if (!getOperand(i)->isLoopInvariant(QueryLoop))
345 return false;
346
347 // Otherwise it's loop-invariant.
348 return true;
Chris Lattner53e677a2004-04-02 20:23:17 +0000349}
350
Dan Gohman39125d82010-02-13 00:19:39 +0000351bool
352SCEVAddRecExpr::dominates(BasicBlock *BB, DominatorTree *DT) const {
353 return DT->dominates(L->getHeader(), BB) &&
354 SCEVNAryExpr::dominates(BB, DT);
355}
356
357bool
358SCEVAddRecExpr::properlyDominates(BasicBlock *BB, DominatorTree *DT) const {
359 // This uses a "dominates" query instead of "properly dominates" query because
360 // the instruction which produces the addrec's value is a PHI, and a PHI
361 // effectively properly dominates its entire containing block.
362 return DT->dominates(L->getHeader(), BB) &&
363 SCEVNAryExpr::properlyDominates(BB, DT);
364}
365
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000366void SCEVAddRecExpr::print(raw_ostream &OS) const {
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000367 OS << "{" << *Operands[0];
Dan Gohmanf9e64722010-03-18 01:17:13 +0000368 for (unsigned i = 1, e = NumOperands; i != e; ++i)
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000369 OS << ",+," << *Operands[i];
Dan Gohman30733292010-01-09 18:17:45 +0000370 OS << "}<";
371 WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false);
372 OS << ">";
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000373}
Chris Lattner53e677a2004-04-02 20:23:17 +0000374
Dan Gohmanab37f502010-08-02 23:49:30 +0000375void SCEVUnknown::deleted() {
376 // Clear this SCEVUnknown from ValuesAtScopes.
377 SE->ValuesAtScopes.erase(this);
378
379 // Remove this SCEVUnknown from the uniquing map.
380 SE->UniqueSCEVs.RemoveNode(this);
381
382 // Release the value.
383 setValPtr(0);
384}
385
386void SCEVUnknown::allUsesReplacedWith(Value *New) {
387 // Clear this SCEVUnknown from ValuesAtScopes.
388 SE->ValuesAtScopes.erase(this);
389
390 // Remove this SCEVUnknown from the uniquing map.
391 SE->UniqueSCEVs.RemoveNode(this);
392
393 // Update this SCEVUnknown to point to the new value. This is needed
394 // because there may still be outstanding SCEVs which still point to
395 // this SCEVUnknown.
396 setValPtr(New);
397}
398
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000399bool SCEVUnknown::isLoopInvariant(const Loop *L) const {
400 // All non-instruction values are loop invariant. All instructions are loop
401 // invariant if they are not contained in the specified loop.
Dan Gohmana3035a62009-05-20 01:01:24 +0000402 // Instructions are never considered invariant in the function body
403 // (null loop) because they are defined within the "loop".
Dan Gohmanab37f502010-08-02 23:49:30 +0000404 if (Instruction *I = dyn_cast<Instruction>(getValue()))
Dan Gohman92329c72009-12-18 01:24:09 +0000405 return L && !L->contains(I);
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000406 return true;
407}
Chris Lattner53e677a2004-04-02 20:23:17 +0000408
Evan Cheng5a6c1a82009-02-17 00:13:06 +0000409bool SCEVUnknown::dominates(BasicBlock *BB, DominatorTree *DT) const {
410 if (Instruction *I = dyn_cast<Instruction>(getValue()))
411 return DT->dominates(I->getParent(), BB);
412 return true;
413}
414
Dan Gohman6e70e312009-09-27 15:26:03 +0000415bool SCEVUnknown::properlyDominates(BasicBlock *BB, DominatorTree *DT) const {
416 if (Instruction *I = dyn_cast<Instruction>(getValue()))
417 return DT->properlyDominates(I->getParent(), BB);
418 return true;
419}
420
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000421const Type *SCEVUnknown::getType() const {
Dan Gohmanab37f502010-08-02 23:49:30 +0000422 return getValue()->getType();
Chris Lattner0a7f98c2004-04-15 15:07:24 +0000423}
Chris Lattner53e677a2004-04-02 20:23:17 +0000424
Dan Gohman0f5efe52010-01-28 02:15:55 +0000425bool SCEVUnknown::isSizeOf(const Type *&AllocTy) const {
Dan Gohmanab37f502010-08-02 23:49:30 +0000426 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
Dan Gohman0f5efe52010-01-28 02:15:55 +0000427 if (VCE->getOpcode() == Instruction::PtrToInt)
428 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
Dan Gohman8db08df2010-02-02 01:38:49 +0000429 if (CE->getOpcode() == Instruction::GetElementPtr &&
430 CE->getOperand(0)->isNullValue() &&
431 CE->getNumOperands() == 2)
432 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1)))
433 if (CI->isOne()) {
434 AllocTy = cast<PointerType>(CE->getOperand(0)->getType())
435 ->getElementType();
436 return true;
437 }
Dan Gohman0f5efe52010-01-28 02:15:55 +0000438
439 return false;
440}
441
442bool SCEVUnknown::isAlignOf(const Type *&AllocTy) const {
Dan Gohmanab37f502010-08-02 23:49:30 +0000443 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
Dan Gohman0f5efe52010-01-28 02:15:55 +0000444 if (VCE->getOpcode() == Instruction::PtrToInt)
445 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
Dan Gohman8db08df2010-02-02 01:38:49 +0000446 if (CE->getOpcode() == Instruction::GetElementPtr &&
447 CE->getOperand(0)->isNullValue()) {
448 const Type *Ty =
449 cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
450 if (const StructType *STy = dyn_cast<StructType>(Ty))
451 if (!STy->isPacked() &&
452 CE->getNumOperands() == 3 &&
453 CE->getOperand(1)->isNullValue()) {
454 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2)))
455 if (CI->isOne() &&
456 STy->getNumElements() == 2 &&
Duncan Sandsb0bc6c32010-02-15 16:12:20 +0000457 STy->getElementType(0)->isIntegerTy(1)) {
Dan Gohman8db08df2010-02-02 01:38:49 +0000458 AllocTy = STy->getElementType(1);
459 return true;
460 }
461 }
462 }
Dan Gohman0f5efe52010-01-28 02:15:55 +0000463
464 return false;
465}
466
Dan Gohman4f8eea82010-02-01 18:27:38 +0000467bool SCEVUnknown::isOffsetOf(const Type *&CTy, Constant *&FieldNo) const {
Dan Gohmanab37f502010-08-02 23:49:30 +0000468 if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
Dan Gohman4f8eea82010-02-01 18:27:38 +0000469 if (VCE->getOpcode() == Instruction::PtrToInt)
470 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
471 if (CE->getOpcode() == Instruction::GetElementPtr &&
472 CE->getNumOperands() == 3 &&
473 CE->getOperand(0)->isNullValue() &&
474 CE->getOperand(1)->isNullValue()) {
475 const Type *Ty =
476 cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
477 // Ignore vector types here so that ScalarEvolutionExpander doesn't
478 // emit getelementptrs that index into vectors.
Duncan Sands1df98592010-02-16 11:11:14 +0000479 if (Ty->isStructTy() || Ty->isArrayTy()) {
Dan Gohman4f8eea82010-02-01 18:27:38 +0000480 CTy = Ty;
481 FieldNo = CE->getOperand(2);
482 return true;
483 }
484 }
485
486 return false;
487}
488
Dan Gohmanb7ef7292009-04-21 00:47:46 +0000489void SCEVUnknown::print(raw_ostream &OS) const {
Dan Gohman0f5efe52010-01-28 02:15:55 +0000490 const Type *AllocTy;
491 if (isSizeOf(AllocTy)) {
492 OS << "sizeof(" << *AllocTy << ")";
493 return;
494 }
495 if (isAlignOf(AllocTy)) {
496 OS << "alignof(" << *AllocTy << ")";
497 return;
498 }
499
Dan Gohman4f8eea82010-02-01 18:27:38 +0000500 const Type *CTy;
Dan Gohman0f5efe52010-01-28 02:15:55 +0000501 Constant *FieldNo;
Dan Gohman4f8eea82010-02-01 18:27:38 +0000502 if (isOffsetOf(CTy, FieldNo)) {
503 OS << "offsetof(" << *CTy << ", ";
Dan Gohman0f5efe52010-01-28 02:15:55 +0000504 WriteAsOperand(OS, FieldNo, false);
505 OS << ")";
506 return;
507 }
508
509 // Otherwise just print it normally.
Dan Gohmanab37f502010-08-02 23:49:30 +0000510 WriteAsOperand(OS, getValue(), false);
Chris Lattner53e677a2004-04-02 20:23:17 +0000511}
512
Chris Lattner8d741b82004-06-20 06:23:15 +0000513//===----------------------------------------------------------------------===//
514// SCEV Utilities
515//===----------------------------------------------------------------------===//
516
517namespace {
518 /// SCEVComplexityCompare - Return true if the complexity of the LHS is less
519 /// than the complexity of the RHS. This comparator is used to canonicalize
520 /// expressions.
Nick Lewycky6726b6d2009-10-25 06:33:48 +0000521 class SCEVComplexityCompare {
Dan Gohman9f1fb422010-08-13 20:17:27 +0000522 const LoopInfo *const LI;
Dan Gohman72861302009-05-07 14:39:04 +0000523 public:
Dan Gohmane72079a2010-07-23 21:18:55 +0000524 explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {}
Dan Gohman72861302009-05-07 14:39:04 +0000525
Dan Gohmanf7b37b22008-04-14 18:23:56 +0000526 bool operator()(const SCEV *LHS, const SCEV *RHS) const {
Dan Gohman42214892009-08-31 21:15:23 +0000527 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
528 if (LHS == RHS)
529 return false;
530
Dan Gohman72861302009-05-07 14:39:04 +0000531 // Primarily, sort the SCEVs by their getSCEVType().
Dan Gohman304a7a62010-07-23 21:20:52 +0000532 unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
533 if (LType != RType)
534 return LType < RType;
Dan Gohman72861302009-05-07 14:39:04 +0000535
Dan Gohman3bf63762010-06-18 19:54:20 +0000536 // Aside from the getSCEVType() ordering, the particular ordering
537 // isn't very important except that it's beneficial to be consistent,
538 // so that (a + b) and (b + a) don't end up as different expressions.
539
540 // Sort SCEVUnknown values with some loose heuristics. TODO: This is
541 // not as complete as it could be.
542 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(LHS)) {
543 const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000544 const Value *LV = LU->getValue(), *RV = RU->getValue();
Dan Gohman3bf63762010-06-18 19:54:20 +0000545
546 // Order pointer values after integer values. This helps SCEVExpander
547 // form GEPs.
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000548 bool LIsPointer = LV->getType()->isPointerTy(),
549 RIsPointer = RV->getType()->isPointerTy();
Dan Gohman304a7a62010-07-23 21:20:52 +0000550 if (LIsPointer != RIsPointer)
551 return RIsPointer;
Dan Gohman3bf63762010-06-18 19:54:20 +0000552
553 // Compare getValueID values.
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000554 unsigned LID = LV->getValueID(),
555 RID = RV->getValueID();
Dan Gohman304a7a62010-07-23 21:20:52 +0000556 if (LID != RID)
557 return LID < RID;
Dan Gohman3bf63762010-06-18 19:54:20 +0000558
559 // Sort arguments by their position.
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000560 if (const Argument *LA = dyn_cast<Argument>(LV)) {
561 const Argument *RA = cast<Argument>(RV);
Dan Gohman3bf63762010-06-18 19:54:20 +0000562 return LA->getArgNo() < RA->getArgNo();
563 }
564
565 // For instructions, compare their loop depth, and their opcode.
566 // This is pretty loose.
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000567 if (const Instruction *LInst = dyn_cast<Instruction>(LV)) {
568 const Instruction *RInst = cast<Instruction>(RV);
Dan Gohman3bf63762010-06-18 19:54:20 +0000569
570 // Compare loop depths.
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000571 const BasicBlock *LParent = LInst->getParent(),
572 *RParent = RInst->getParent();
573 if (LParent != RParent) {
574 unsigned LDepth = LI->getLoopDepth(LParent),
575 RDepth = LI->getLoopDepth(RParent);
576 if (LDepth != RDepth)
577 return LDepth < RDepth;
578 }
Dan Gohman3bf63762010-06-18 19:54:20 +0000579
580 // Compare the number of operands.
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000581 unsigned LNumOps = LInst->getNumOperands(),
582 RNumOps = RInst->getNumOperands();
Dan Gohman304a7a62010-07-23 21:20:52 +0000583 if (LNumOps != RNumOps)
584 return LNumOps < RNumOps;
Dan Gohman3bf63762010-06-18 19:54:20 +0000585 }
586
587 return false;
588 }
589
590 // Compare constant values.
591 if (const SCEVConstant *LC = dyn_cast<SCEVConstant>(LHS)) {
592 const SCEVConstant *RC = cast<SCEVConstant>(RHS);
Dan Gohmane28d7922010-08-16 16:25:35 +0000593 const APInt &LA = LC->getValue()->getValue();
594 const APInt &RA = RC->getValue()->getValue();
595 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
Dan Gohman304a7a62010-07-23 21:20:52 +0000596 if (LBitWidth != RBitWidth)
597 return LBitWidth < RBitWidth;
Dan Gohmane28d7922010-08-16 16:25:35 +0000598 return LA.ult(RA);
Dan Gohman3bf63762010-06-18 19:54:20 +0000599 }
600
601 // Compare addrec loop depths.
602 if (const SCEVAddRecExpr *LA = dyn_cast<SCEVAddRecExpr>(LHS)) {
603 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS);
Dan Gohman0ad2c7a2010-08-13 21:24:58 +0000604 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
605 if (LLoop != RLoop) {
606 unsigned LDepth = LLoop->getLoopDepth(),
607 RDepth = RLoop->getLoopDepth();
608 if (LDepth != RDepth)
609 return LDepth < RDepth;
610 }
Dan Gohman3bf63762010-06-18 19:54:20 +0000611 }
612
613 // Lexicographically compare n-ary expressions.
614 if (const SCEVNAryExpr *LC = dyn_cast<SCEVNAryExpr>(LHS)) {
615 const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS);
Dan Gohman304a7a62010-07-23 21:20:52 +0000616 unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands();
617 for (unsigned i = 0; i != LNumOps; ++i) {
618 if (i >= RNumOps)
Dan Gohman3bf63762010-06-18 19:54:20 +0000619 return false;
Dan Gohman304a7a62010-07-23 21:20:52 +0000620 const SCEV *LOp = LC->getOperand(i), *ROp = RC->getOperand(i);
621 if (operator()(LOp, ROp))
Dan Gohman3bf63762010-06-18 19:54:20 +0000622 return true;
Dan Gohman304a7a62010-07-23 21:20:52 +0000623 if (operator()(ROp, LOp))
Dan Gohman3bf63762010-06-18 19:54:20 +0000624 return false;
625 }
Dan Gohman304a7a62010-07-23 21:20:52 +0000626 return LNumOps < RNumOps;
Dan Gohman3bf63762010-06-18 19:54:20 +0000627 }
628
629 // Lexicographically compare udiv expressions.
630 if (const SCEVUDivExpr *LC = dyn_cast<SCEVUDivExpr>(LHS)) {
631 const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS);
Dan Gohman304a7a62010-07-23 21:20:52 +0000632 const SCEV *LL = LC->getLHS(), *LR = LC->getRHS(),
633 *RL = RC->getLHS(), *RR = RC->getRHS();
634 if (operator()(LL, RL))
Dan Gohman3bf63762010-06-18 19:54:20 +0000635 return true;
Dan Gohman304a7a62010-07-23 21:20:52 +0000636 if (operator()(RL, LL))
Dan Gohman3bf63762010-06-18 19:54:20 +0000637 return false;
Dan Gohman304a7a62010-07-23 21:20:52 +0000638 if (operator()(LR, RR))
Dan Gohman3bf63762010-06-18 19:54:20 +0000639 return true;
Dan Gohman304a7a62010-07-23 21:20:52 +0000640 if (operator()(RR, LR))
Dan Gohman3bf63762010-06-18 19:54:20 +0000641 return false;
642 return false;
643 }
644
645 // Compare cast expressions by operand.
646 if (const SCEVCastExpr *LC = dyn_cast<SCEVCastExpr>(LHS)) {
647 const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS);
648 return operator()(LC->getOperand(), RC->getOperand());
649 }
650
651 llvm_unreachable("Unknown SCEV kind!");
652 return false;
Chris Lattner8d741b82004-06-20 06:23:15 +0000653 }
654 };
655}
656
657/// GroupByComplexity - Given a list of SCEV objects, order them by their
658/// complexity, and group objects of the same complexity together by value.
659/// When this routine is finished, we know that any duplicates in the vector are
660/// consecutive and that complexity is monotonically increasing.
661///
Dan Gohman3f46a3a2010-03-01 17:49:51 +0000662/// Note that we go take special precautions to ensure that we get deterministic
Chris Lattner8d741b82004-06-20 06:23:15 +0000663/// results from this routine. In other words, we don't want the results of
664/// this to depend on where the addresses of various SCEV objects happened to
665/// land in memory.
666///
Dan Gohman0bba49c2009-07-07 17:06:11 +0000667static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
Dan Gohman72861302009-05-07 14:39:04 +0000668 LoopInfo *LI) {
Chris Lattner8d741b82004-06-20 06:23:15 +0000669 if (Ops.size() < 2) return; // Noop
670 if (Ops.size() == 2) {
671 // This is the common case, which also happens to be trivially simple.
672 // Special case it.
Dan Gohman3bf63762010-06-18 19:54:20 +0000673 if (SCEVComplexityCompare(LI)(Ops[1], Ops[0]))
Chris Lattner8d741b82004-06-20 06:23:15 +0000674 std::swap(Ops[0], Ops[1]);
675 return;
676 }
677
Dan Gohman3bf63762010-06-18 19:54:20 +0000678 // Do the rough sort by complexity.
679 std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI));
680
681 // Now that we are sorted by complexity, group elements of the same
682 // complexity. Note that this is, at worst, N^2, but the vector is likely to
683 // be extremely short in practice. Note that we take this approach because we
684 // do not want to depend on the addresses of the objects we are grouping.
685 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
686 const SCEV *S = Ops[i];
687 unsigned Complexity = S->getSCEVType();
688
689 // If there are any objects of the same complexity and same value as this
690 // one, group them.
691 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
692 if (Ops[j] == S) { // Found a duplicate.
693 // Move it to immediately after i'th element.
694 std::swap(Ops[i+1], Ops[j]);
695 ++i; // no need to rescan it.
696 if (i == e-2) return; // Done!
697 }
698 }
699 }
Chris Lattner8d741b82004-06-20 06:23:15 +0000700}
701
Chris Lattner53e677a2004-04-02 20:23:17 +0000702
Chris Lattner53e677a2004-04-02 20:23:17 +0000703
704//===----------------------------------------------------------------------===//
705// Simple SCEV method implementations
706//===----------------------------------------------------------------------===//
707
Eli Friedmanb42a6262008-08-04 23:49:06 +0000708/// BinomialCoefficient - Compute BC(It, K). The result has width W.
Dan Gohman6c0866c2009-05-24 23:45:28 +0000709/// Assume, K > 0.
Dan Gohman0bba49c2009-07-07 17:06:11 +0000710static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
Dan Gohmanc2b015e2009-07-21 00:38:55 +0000711 ScalarEvolution &SE,
712 const Type* ResultTy) {
Eli Friedmanb42a6262008-08-04 23:49:06 +0000713 // Handle the simplest case efficiently.
714 if (K == 1)
715 return SE.getTruncateOrZeroExtend(It, ResultTy);
716
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000717 // We are using the following formula for BC(It, K):
718 //
719 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
720 //
Eli Friedmanb42a6262008-08-04 23:49:06 +0000721 // Suppose, W is the bitwidth of the return value. We must be prepared for
722 // overflow. Hence, we must assure that the result of our computation is
723 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
724 // safe in modular arithmetic.
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000725 //
Eli Friedmanb42a6262008-08-04 23:49:06 +0000726 // However, this code doesn't use exactly that formula; the formula it uses
Dan Gohman64a845e2009-06-24 04:48:43 +0000727 // is something like the following, where T is the number of factors of 2 in
Eli Friedmanb42a6262008-08-04 23:49:06 +0000728 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
729 // exponentiation:
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000730 //
Eli Friedmanb42a6262008-08-04 23:49:06 +0000731 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000732 //
Eli Friedmanb42a6262008-08-04 23:49:06 +0000733 // This formula is trivially equivalent to the previous formula. However,
734 // this formula can be implemented much more efficiently. The trick is that
735 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
736 // arithmetic. To do exact division in modular arithmetic, all we have
737 // to do is multiply by the inverse. Therefore, this step can be done at
738 // width W.
Dan Gohman64a845e2009-06-24 04:48:43 +0000739 //
Eli Friedmanb42a6262008-08-04 23:49:06 +0000740 // The next issue is how to safely do the division by 2^T. The way this
741 // is done is by doing the multiplication step at a width of at least W + T
742 // bits. This way, the bottom W+T bits of the product are accurate. Then,
743 // when we perform the division by 2^T (which is equivalent to a right shift
744 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
745 // truncated out after the division by 2^T.
746 //
747 // In comparison to just directly using the first formula, this technique
748 // is much more efficient; using the first formula requires W * K bits,
749 // but this formula less than W + K bits. Also, the first formula requires
750 // a division step, whereas this formula only requires multiplies and shifts.
751 //
752 // It doesn't matter whether the subtraction step is done in the calculation
753 // width or the input iteration count's width; if the subtraction overflows,
754 // the result must be zero anyway. We prefer here to do it in the width of
755 // the induction variable because it helps a lot for certain cases; CodeGen
756 // isn't smart enough to ignore the overflow, which leads to much less
757 // efficient code if the width of the subtraction is wider than the native
758 // register width.
759 //
760 // (It's possible to not widen at all by pulling out factors of 2 before
761 // the multiplication; for example, K=2 can be calculated as
762 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
763 // extra arithmetic, so it's not an obvious win, and it gets
764 // much more complicated for K > 3.)
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000765
Eli Friedmanb42a6262008-08-04 23:49:06 +0000766 // Protection from insane SCEVs; this bound is conservative,
767 // but it probably doesn't matter.
768 if (K > 1000)
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +0000769 return SE.getCouldNotCompute();
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000770
Dan Gohmanaf79fb52009-04-21 01:07:12 +0000771 unsigned W = SE.getTypeSizeInBits(ResultTy);
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000772
Eli Friedmanb42a6262008-08-04 23:49:06 +0000773 // Calculate K! / 2^T and T; we divide out the factors of two before
774 // multiplying for calculating K! / 2^T to avoid overflow.
775 // Other overflow doesn't matter because we only care about the bottom
776 // W bits of the result.
777 APInt OddFactorial(W, 1);
778 unsigned T = 1;
779 for (unsigned i = 3; i <= K; ++i) {
780 APInt Mult(W, i);
781 unsigned TwoFactors = Mult.countTrailingZeros();
782 T += TwoFactors;
783 Mult = Mult.lshr(TwoFactors);
784 OddFactorial *= Mult;
Chris Lattner53e677a2004-04-02 20:23:17 +0000785 }
Nick Lewycky6f8abf92008-06-13 04:38:55 +0000786
Eli Friedmanb42a6262008-08-04 23:49:06 +0000787 // We need at least W + T bits for the multiplication step
Nick Lewycky237d8732009-01-25 08:16:27 +0000788 unsigned CalculationBits = W + T;
Eli Friedmanb42a6262008-08-04 23:49:06 +0000789
Dan Gohman3f46a3a2010-03-01 17:49:51 +0000790 // Calculate 2^T, at width T+W.
Eli Friedmanb42a6262008-08-04 23:49:06 +0000791 APInt DivFactor = APInt(CalculationBits, 1).shl(T);
792
793 // Calculate the multiplicative inverse of K! / 2^T;
794 // this multiplication factor will perform the exact division by
795 // K! / 2^T.
796 APInt Mod = APInt::getSignedMinValue(W+1);
797 APInt MultiplyFactor = OddFactorial.zext(W+1);
798 MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod);
799 MultiplyFactor = MultiplyFactor.trunc(W);
800
801 // Calculate the product, at width T+W
Owen Anderson1d0be152009-08-13 21:58:54 +0000802 const IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
803 CalculationBits);
Dan Gohman0bba49c2009-07-07 17:06:11 +0000804 const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
Eli Friedmanb42a6262008-08-04 23:49:06 +0000805 for (unsigned i = 1; i != K; ++i) {
Dan Gohmandeff6212010-05-03 22:09:21 +0000806 const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
Eli Friedmanb42a6262008-08-04 23:49:06 +0000807 Dividend = SE.getMulExpr(Dividend,
808 SE.getTruncateOrZeroExtend(S, CalculationTy));
809 }
810
811 // Divide by 2^T
Dan Gohman0bba49c2009-07-07 17:06:11 +0000812 const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
Eli Friedmanb42a6262008-08-04 23:49:06 +0000813
814 // Truncate the result, and divide by K! / 2^T.
815
816 return SE.getMulExpr(SE.getConstant(MultiplyFactor),
817 SE.getTruncateOrZeroExtend(DivResult, ResultTy));
Chris Lattner53e677a2004-04-02 20:23:17 +0000818}
819
Chris Lattner53e677a2004-04-02 20:23:17 +0000820/// evaluateAtIteration - Return the value of this chain of recurrences at
821/// the specified iteration number. We can evaluate this recurrence by
822/// multiplying each element in the chain by the binomial coefficient
823/// corresponding to it. In other words, we can evaluate {A,+,B,+,C,+,D} as:
824///
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000825/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
Chris Lattner53e677a2004-04-02 20:23:17 +0000826///
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000827/// where BC(It, k) stands for binomial coefficient.
Chris Lattner53e677a2004-04-02 20:23:17 +0000828///
Dan Gohman0bba49c2009-07-07 17:06:11 +0000829const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
Dan Gohmanc2b015e2009-07-21 00:38:55 +0000830 ScalarEvolution &SE) const {
Dan Gohman0bba49c2009-07-07 17:06:11 +0000831 const SCEV *Result = getStart();
Chris Lattner53e677a2004-04-02 20:23:17 +0000832 for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {
Wojciech Matyjewicze3320a12008-02-11 11:03:14 +0000833 // The computation is correct in the face of overflow provided that the
834 // multiplication is performed _after_ the evaluation of the binomial
835 // coefficient.
Dan Gohman0bba49c2009-07-07 17:06:11 +0000836 const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType());
Nick Lewyckycb8f1b52008-10-13 03:58:02 +0000837 if (isa<SCEVCouldNotCompute>(Coeff))
838 return Coeff;
839
840 Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff));
Chris Lattner53e677a2004-04-02 20:23:17 +0000841 }
842 return Result;
843}
844
Chris Lattner53e677a2004-04-02 20:23:17 +0000845//===----------------------------------------------------------------------===//
846// SCEV Expression folder implementations
847//===----------------------------------------------------------------------===//
848
Dan Gohman0bba49c2009-07-07 17:06:11 +0000849const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op,
Dan Gohmanf5074ec2009-07-13 22:05:32 +0000850 const Type *Ty) {
Dan Gohmanaf79fb52009-04-21 01:07:12 +0000851 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
Dan Gohmanfb17fd22009-04-21 00:55:22 +0000852 "This is not a truncating conversion!");
Dan Gohman10b94792009-05-01 16:44:18 +0000853 assert(isSCEVable(Ty) &&
854 "This is not a conversion to a SCEVable type!");
855 Ty = getEffectiveSCEVType(Ty);
Dan Gohmanfb17fd22009-04-21 00:55:22 +0000856
Dan Gohmanc050fd92009-07-13 20:50:19 +0000857 FoldingSetNodeID ID;
858 ID.AddInteger(scTruncate);
859 ID.AddPointer(Op);
860 ID.AddPointer(Ty);
861 void *IP = 0;
862 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
863
Dan Gohmanc39f44b2009-06-30 20:13:32 +0000864 // Fold if the operand is constant.
Dan Gohman622ed672009-05-04 22:02:23 +0000865 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
Dan Gohmanb8be8b72009-06-24 00:38:39 +0000866 return getConstant(
Dan Gohman1faa8822010-06-24 16:33:38 +0000867 cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(),
868 getEffectiveSCEVType(Ty))));
Chris Lattner53e677a2004-04-02 20:23:17 +0000869
Dan Gohman20900ca2009-04-22 16:20:48 +0000870 // trunc(trunc(x)) --> trunc(x)
Dan Gohman622ed672009-05-04 22:02:23 +0000871 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op))
Dan Gohman20900ca2009-04-22 16:20:48 +0000872 return getTruncateExpr(ST->getOperand(), Ty);
873
Nick Lewycky5cd28fa2009-04-23 05:15:08 +0000874 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
Dan Gohman622ed672009-05-04 22:02:23 +0000875 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
Nick Lewycky5cd28fa2009-04-23 05:15:08 +0000876 return getTruncateOrSignExtend(SS->getOperand(), Ty);
877
878 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
Dan Gohman622ed672009-05-04 22:02:23 +0000879 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
Nick Lewycky5cd28fa2009-04-23 05:15:08 +0000880 return getTruncateOrZeroExtend(SZ->getOperand(), Ty);
881
Dan Gohman6864db62009-06-18 16:24:47 +0000882 // If the input value is a chrec scev, truncate the chrec's operands.
Dan Gohman622ed672009-05-04 22:02:23 +0000883 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
Dan Gohman0bba49c2009-07-07 17:06:11 +0000884 SmallVector<const SCEV *, 4> Operands;
Chris Lattner53e677a2004-04-02 20:23:17 +0000885 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
Dan Gohman728c7f32009-05-08 21:03:19 +0000886 Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty));
887 return getAddRecExpr(Operands, AddRec->getLoop());
Chris Lattner53e677a2004-04-02 20:23:17 +0000888 }
889
Dan Gohmanf53462d2010-07-15 20:02:11 +0000890 // As a special case, fold trunc(undef) to undef. We don't want to
891 // know too much about SCEVUnknowns, but this special case is handy
892 // and harmless.
893 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Op))
894 if (isa<UndefValue>(U->getValue()))
895 return getSCEV(UndefValue::get(Ty));
896
Dan Gohman420ab912010-06-25 18:47:08 +0000897 // The cast wasn't folded; create an explicit cast node. We can reuse
898 // the existing insert position since if we get here, we won't have
899 // made any changes which would invalidate it.
Dan Gohman95531882010-03-18 18:49:47 +0000900 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
901 Op, Ty);
Dan Gohman1c343752009-06-27 21:21:31 +0000902 UniqueSCEVs.InsertNode(S, IP);
903 return S;
Chris Lattner53e677a2004-04-02 20:23:17 +0000904}
905
Dan Gohman0bba49c2009-07-07 17:06:11 +0000906const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
Dan Gohmanf5074ec2009-07-13 22:05:32 +0000907 const Type *Ty) {
Dan Gohmanaf79fb52009-04-21 01:07:12 +0000908 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
Dan Gohman8170a682009-04-16 19:25:55 +0000909 "This is not an extending conversion!");
Dan Gohman10b94792009-05-01 16:44:18 +0000910 assert(isSCEVable(Ty) &&
911 "This is not a conversion to a SCEVable type!");
912 Ty = getEffectiveSCEVType(Ty);
Dan Gohman8170a682009-04-16 19:25:55 +0000913
Dan Gohmanc39f44b2009-06-30 20:13:32 +0000914 // Fold if the operand is constant.
Dan Gohmaneaf6cf22010-06-24 16:47:03 +0000915 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
916 return getConstant(
917 cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(),
918 getEffectiveSCEVType(Ty))));
Chris Lattner53e677a2004-04-02 20:23:17 +0000919
Dan Gohman20900ca2009-04-22 16:20:48 +0000920 // zext(zext(x)) --> zext(x)
Dan Gohman622ed672009-05-04 22:02:23 +0000921 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
Dan Gohman20900ca2009-04-22 16:20:48 +0000922 return getZeroExtendExpr(SZ->getOperand(), Ty);
923
Dan Gohman69fbc7f2009-07-13 20:55:53 +0000924 // Before doing any expensive analysis, check to see if we've already
925 // computed a SCEV for this Op and Ty.
926 FoldingSetNodeID ID;
927 ID.AddInteger(scZeroExtend);
928 ID.AddPointer(Op);
929 ID.AddPointer(Ty);
930 void *IP = 0;
931 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
932
Dan Gohman01ecca22009-04-27 20:16:15 +0000933 // If the input value is a chrec scev, and we can prove that the value
Chris Lattner53e677a2004-04-02 20:23:17 +0000934 // did not overflow the old, smaller, value, we can zero extend all of the
Dan Gohman01ecca22009-04-27 20:16:15 +0000935 // operands (often constants). This allows analysis of something like
Chris Lattner53e677a2004-04-02 20:23:17 +0000936 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
Dan Gohman622ed672009-05-04 22:02:23 +0000937 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
Dan Gohman01ecca22009-04-27 20:16:15 +0000938 if (AR->isAffine()) {
Dan Gohman85b05a22009-07-13 21:35:55 +0000939 const SCEV *Start = AR->getStart();
940 const SCEV *Step = AR->getStepRecurrence(*this);
941 unsigned BitWidth = getTypeSizeInBits(AR->getType());
942 const Loop *L = AR->getLoop();
943
Dan Gohmaneb490a72009-07-25 01:22:26 +0000944 // If we have special knowledge that this addrec won't overflow,
945 // we don't need to do any further analysis.
Dan Gohman5078f842009-08-20 17:11:38 +0000946 if (AR->hasNoUnsignedWrap())
Dan Gohmaneb490a72009-07-25 01:22:26 +0000947 return getAddRecExpr(getZeroExtendExpr(Start, Ty),
948 getZeroExtendExpr(Step, Ty),
949 L);
950
Dan Gohman01ecca22009-04-27 20:16:15 +0000951 // Check whether the backedge-taken count is SCEVCouldNotCompute.
952 // Note that this serves two purposes: It filters out loops that are
953 // simply not analyzable, and it covers the case where this code is
954 // being called from within backedge-taken count analysis, such that
955 // attempting to ask for the backedge-taken count would likely result
956 // in infinite recursion. In the later case, the analysis code will
957 // cope with a conservative value, and it will take care to purge
958 // that value once it has finished.
Dan Gohman85b05a22009-07-13 21:35:55 +0000959 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
Dan Gohmana1af7572009-04-30 20:47:05 +0000960 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
Dan Gohmanf0aa4852009-04-29 01:54:20 +0000961 // Manually compute the final value for AR, checking for
Dan Gohmanac70cea2009-04-29 22:28:28 +0000962 // overflow.
Dan Gohman01ecca22009-04-27 20:16:15 +0000963
964 // Check whether the backedge-taken count can be losslessly casted to
965 // the addrec's type. The count is always unsigned.
Dan Gohman0bba49c2009-07-07 17:06:11 +0000966 const SCEV *CastedMaxBECount =
Dan Gohmana1af7572009-04-30 20:47:05 +0000967 getTruncateOrZeroExtend(MaxBECount, Start->getType());
Dan Gohman0bba49c2009-07-07 17:06:11 +0000968 const SCEV *RecastedMaxBECount =
Dan Gohman5183cae2009-05-18 15:58:39 +0000969 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
970 if (MaxBECount == RecastedMaxBECount) {
Owen Anderson1d0be152009-08-13 21:58:54 +0000971 const Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
Dan Gohmana1af7572009-04-30 20:47:05 +0000972 // Check whether Start+Step*MaxBECount has no unsigned overflow.
Dan Gohman8f767d92010-02-24 19:31:06 +0000973 const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step);
Dan Gohman0bba49c2009-07-07 17:06:11 +0000974 const SCEV *Add = getAddExpr(Start, ZMul);
975 const SCEV *OperandExtendedAdd =
Dan Gohman5183cae2009-05-18 15:58:39 +0000976 getAddExpr(getZeroExtendExpr(Start, WideTy),
977 getMulExpr(getZeroExtendExpr(CastedMaxBECount, WideTy),
978 getZeroExtendExpr(Step, WideTy)));
979 if (getZeroExtendExpr(Add, WideTy) == OperandExtendedAdd)
Dan Gohmanac70cea2009-04-29 22:28:28 +0000980 // Return the expression with the addrec on the outside.
981 return getAddRecExpr(getZeroExtendExpr(Start, Ty),
982 getZeroExtendExpr(Step, Ty),
Dan Gohman85b05a22009-07-13 21:35:55 +0000983 L);
Dan Gohman01ecca22009-04-27 20:16:15 +0000984
985 // Similar to above, only this time treat the step value as signed.
986 // This covers loops that count down.
Dan Gohman8f767d92010-02-24 19:31:06 +0000987 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step);
Dan Gohmanac70cea2009-04-29 22:28:28 +0000988 Add = getAddExpr(Start, SMul);
Dan Gohman5183cae2009-05-18 15:58:39 +0000989 OperandExtendedAdd =
990 getAddExpr(getZeroExtendExpr(Start, WideTy),
991 getMulExpr(getZeroExtendExpr(CastedMaxBECount, WideTy),
992 getSignExtendExpr(Step, WideTy)));
993 if (getZeroExtendExpr(Add, WideTy) == OperandExtendedAdd)
Dan Gohmanac70cea2009-04-29 22:28:28 +0000994 // Return the expression with the addrec on the outside.
995 return getAddRecExpr(getZeroExtendExpr(Start, Ty),
996 getSignExtendExpr(Step, Ty),
Dan Gohman85b05a22009-07-13 21:35:55 +0000997 L);
998 }
999
1000 // If the backedge is guarded by a comparison with the pre-inc value
1001 // the addrec is safe. Also, if the entry is guarded by a comparison
1002 // with the start value and the backedge is guarded by a comparison
1003 // with the post-inc value, the addrec is safe.
1004 if (isKnownPositive(Step)) {
1005 const SCEV *N = getConstant(APInt::getMinValue(BitWidth) -
1006 getUnsignedRange(Step).getUnsignedMax());
1007 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) ||
Dan Gohman3948d0b2010-04-11 19:27:13 +00001008 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) &&
Dan Gohman85b05a22009-07-13 21:35:55 +00001009 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT,
1010 AR->getPostIncExpr(*this), N)))
1011 // Return the expression with the addrec on the outside.
1012 return getAddRecExpr(getZeroExtendExpr(Start, Ty),
1013 getZeroExtendExpr(Step, Ty),
1014 L);
1015 } else if (isKnownNegative(Step)) {
1016 const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) -
1017 getSignedRange(Step).getSignedMin());
Dan Gohmanc0ed0092010-05-04 01:11:15 +00001018 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) ||
1019 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) &&
Dan Gohman85b05a22009-07-13 21:35:55 +00001020 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT,
1021 AR->getPostIncExpr(*this), N)))
1022 // Return the expression with the addrec on the outside.
1023 return getAddRecExpr(getZeroExtendExpr(Start, Ty),
1024 getSignExtendExpr(Step, Ty),
1025 L);
Dan Gohman01ecca22009-04-27 20:16:15 +00001026 }
1027 }
1028 }
Chris Lattner53e677a2004-04-02 20:23:17 +00001029
Dan Gohman69fbc7f2009-07-13 20:55:53 +00001030 // The cast wasn't folded; create an explicit cast node.
1031 // Recompute the insert position, as it may have been invalidated.
Dan Gohman1c343752009-06-27 21:21:31 +00001032 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
Dan Gohman95531882010-03-18 18:49:47 +00001033 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
1034 Op, Ty);
Dan Gohman1c343752009-06-27 21:21:31 +00001035 UniqueSCEVs.InsertNode(S, IP);
1036 return S;
Chris Lattner53e677a2004-04-02 20:23:17 +00001037}
1038
Dan Gohman0bba49c2009-07-07 17:06:11 +00001039const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
Dan Gohmanf5074ec2009-07-13 22:05:32 +00001040 const Type *Ty) {
Dan Gohmanaf79fb52009-04-21 01:07:12 +00001041 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
Dan Gohmanfb17fd22009-04-21 00:55:22 +00001042 "This is not an extending conversion!");
Dan Gohman10b94792009-05-01 16:44:18 +00001043 assert(isSCEVable(Ty) &&
1044 "This is not a conversion to a SCEVable type!");
1045 Ty = getEffectiveSCEVType(Ty);
Dan Gohmanfb17fd22009-04-21 00:55:22 +00001046
Dan Gohmanc39f44b2009-06-30 20:13:32 +00001047 // Fold if the operand is constant.
Dan Gohmaneaf6cf22010-06-24 16:47:03 +00001048 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1049 return getConstant(
1050 cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(),
1051 getEffectiveSCEVType(Ty))));
Dan Gohmand19534a2007-06-15 14:38:12 +00001052
Dan Gohman20900ca2009-04-22 16:20:48 +00001053 // sext(sext(x)) --> sext(x)
Dan Gohman622ed672009-05-04 22:02:23 +00001054 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
Dan Gohman20900ca2009-04-22 16:20:48 +00001055 return getSignExtendExpr(SS->getOperand(), Ty);
1056
Dan Gohman69fbc7f2009-07-13 20:55:53 +00001057 // Before doing any expensive analysis, check to see if we've already
1058 // computed a SCEV for this Op and Ty.
1059 FoldingSetNodeID ID;
1060 ID.AddInteger(scSignExtend);
1061 ID.AddPointer(Op);
1062 ID.AddPointer(Ty);
1063 void *IP = 0;
1064 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
1065
Dan Gohman01ecca22009-04-27 20:16:15 +00001066 // If the input value is a chrec scev, and we can prove that the value
Dan Gohmand19534a2007-06-15 14:38:12 +00001067 // did not overflow the old, smaller, value, we can sign extend all of the
Dan Gohman01ecca22009-04-27 20:16:15 +00001068 // operands (often constants). This allows analysis of something like
Dan Gohmand19534a2007-06-15 14:38:12 +00001069 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
Dan Gohman622ed672009-05-04 22:02:23 +00001070 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
Dan Gohman01ecca22009-04-27 20:16:15 +00001071 if (AR->isAffine()) {
Dan Gohman85b05a22009-07-13 21:35:55 +00001072 const SCEV *Start = AR->getStart();
1073 const SCEV *Step = AR->getStepRecurrence(*this);
1074 unsigned BitWidth = getTypeSizeInBits(AR->getType());
1075 const Loop *L = AR->getLoop();
1076
Dan Gohmaneb490a72009-07-25 01:22:26 +00001077 // If we have special knowledge that this addrec won't overflow,
1078 // we don't need to do any further analysis.
Dan Gohman5078f842009-08-20 17:11:38 +00001079 if (AR->hasNoSignedWrap())
Dan Gohmaneb490a72009-07-25 01:22:26 +00001080 return getAddRecExpr(getSignExtendExpr(Start, Ty),
1081 getSignExtendExpr(Step, Ty),
1082 L);
1083
Dan Gohman01ecca22009-04-27 20:16:15 +00001084 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1085 // Note that this serves two purposes: It filters out loops that are
1086 // simply not analyzable, and it covers the case where this code is
1087 // being called from within backedge-taken count analysis, such that
1088 // attempting to ask for the backedge-taken count would likely result
1089 // in infinite recursion. In the later case, the analysis code will
1090 // cope with a conservative value, and it will take care to purge
1091 // that value once it has finished.
Dan Gohman85b05a22009-07-13 21:35:55 +00001092 const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
Dan Gohmana1af7572009-04-30 20:47:05 +00001093 if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
Dan Gohmanf0aa4852009-04-29 01:54:20 +00001094 // Manually compute the final value for AR, checking for
Dan Gohmanac70cea2009-04-29 22:28:28 +00001095 // overflow.
Dan Gohman01ecca22009-04-27 20:16:15 +00001096
1097 // Check whether the backedge-taken count can be losslessly casted to
Dan Gohmanac70cea2009-04-29 22:28:28 +00001098 // the addrec's type. The count is always unsigned.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001099 const SCEV *CastedMaxBECount =
Dan Gohmana1af7572009-04-30 20:47:05 +00001100 getTruncateOrZeroExtend(MaxBECount, Start->getType());
Dan Gohman0bba49c2009-07-07 17:06:11 +00001101 const SCEV *RecastedMaxBECount =
Dan Gohman5183cae2009-05-18 15:58:39 +00001102 getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
1103 if (MaxBECount == RecastedMaxBECount) {
Owen Anderson1d0be152009-08-13 21:58:54 +00001104 const Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
Dan Gohmana1af7572009-04-30 20:47:05 +00001105 // Check whether Start+Step*MaxBECount has no signed overflow.
Dan Gohman8f767d92010-02-24 19:31:06 +00001106 const SCEV *SMul = getMulExpr(CastedMaxBECount, Step);
Dan Gohman0bba49c2009-07-07 17:06:11 +00001107 const SCEV *Add = getAddExpr(Start, SMul);
1108 const SCEV *OperandExtendedAdd =
Dan Gohman5183cae2009-05-18 15:58:39 +00001109 getAddExpr(getSignExtendExpr(Start, WideTy),
1110 getMulExpr(getZeroExtendExpr(CastedMaxBECount, WideTy),
1111 getSignExtendExpr(Step, WideTy)));
1112 if (getSignExtendExpr(Add, WideTy) == OperandExtendedAdd)
Dan Gohmanac70cea2009-04-29 22:28:28 +00001113 // Return the expression with the addrec on the outside.
1114 return getAddRecExpr(getSignExtendExpr(Start, Ty),
1115 getSignExtendExpr(Step, Ty),
Dan Gohman85b05a22009-07-13 21:35:55 +00001116 L);
Dan Gohman850f7912009-07-16 17:34:36 +00001117
1118 // Similar to above, only this time treat the step value as unsigned.
1119 // This covers loops that count up with an unsigned step.
Dan Gohman8f767d92010-02-24 19:31:06 +00001120 const SCEV *UMul = getMulExpr(CastedMaxBECount, Step);
Dan Gohman850f7912009-07-16 17:34:36 +00001121 Add = getAddExpr(Start, UMul);
1122 OperandExtendedAdd =
Dan Gohman19378d62009-07-25 16:03:30 +00001123 getAddExpr(getSignExtendExpr(Start, WideTy),
Dan Gohman850f7912009-07-16 17:34:36 +00001124 getMulExpr(getZeroExtendExpr(CastedMaxBECount, WideTy),
1125 getZeroExtendExpr(Step, WideTy)));
Dan Gohman19378d62009-07-25 16:03:30 +00001126 if (getSignExtendExpr(Add, WideTy) == OperandExtendedAdd)
Dan Gohman850f7912009-07-16 17:34:36 +00001127 // Return the expression with the addrec on the outside.
1128 return getAddRecExpr(getSignExtendExpr(Start, Ty),
1129 getZeroExtendExpr(Step, Ty),
1130 L);
Dan Gohman85b05a22009-07-13 21:35:55 +00001131 }
1132
1133 // If the backedge is guarded by a comparison with the pre-inc value
1134 // the addrec is safe. Also, if the entry is guarded by a comparison
1135 // with the start value and the backedge is guarded by a comparison
1136 // with the post-inc value, the addrec is safe.
1137 if (isKnownPositive(Step)) {
1138 const SCEV *N = getConstant(APInt::getSignedMinValue(BitWidth) -
1139 getSignedRange(Step).getSignedMax());
1140 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_SLT, AR, N) ||
Dan Gohman3948d0b2010-04-11 19:27:13 +00001141 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_SLT, Start, N) &&
Dan Gohman85b05a22009-07-13 21:35:55 +00001142 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_SLT,
1143 AR->getPostIncExpr(*this), N)))
1144 // Return the expression with the addrec on the outside.
1145 return getAddRecExpr(getSignExtendExpr(Start, Ty),
1146 getSignExtendExpr(Step, Ty),
1147 L);
1148 } else if (isKnownNegative(Step)) {
1149 const SCEV *N = getConstant(APInt::getSignedMaxValue(BitWidth) -
1150 getSignedRange(Step).getSignedMin());
1151 if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_SGT, AR, N) ||
Dan Gohman3948d0b2010-04-11 19:27:13 +00001152 (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_SGT, Start, N) &&
Dan Gohman85b05a22009-07-13 21:35:55 +00001153 isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_SGT,
1154 AR->getPostIncExpr(*this), N)))
1155 // Return the expression with the addrec on the outside.
1156 return getAddRecExpr(getSignExtendExpr(Start, Ty),
1157 getSignExtendExpr(Step, Ty),
1158 L);
Dan Gohman01ecca22009-04-27 20:16:15 +00001159 }
1160 }
1161 }
Dan Gohmand19534a2007-06-15 14:38:12 +00001162
Dan Gohman69fbc7f2009-07-13 20:55:53 +00001163 // The cast wasn't folded; create an explicit cast node.
1164 // Recompute the insert position, as it may have been invalidated.
Dan Gohman1c343752009-06-27 21:21:31 +00001165 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
Dan Gohman95531882010-03-18 18:49:47 +00001166 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
1167 Op, Ty);
Dan Gohman1c343752009-06-27 21:21:31 +00001168 UniqueSCEVs.InsertNode(S, IP);
1169 return S;
Dan Gohmand19534a2007-06-15 14:38:12 +00001170}
1171
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00001172/// getAnyExtendExpr - Return a SCEV for the given operand extended with
1173/// unspecified bits out to the given type.
1174///
Dan Gohman0bba49c2009-07-07 17:06:11 +00001175const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
Dan Gohmanc40f17b2009-08-18 16:46:41 +00001176 const Type *Ty) {
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00001177 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1178 "This is not an extending conversion!");
1179 assert(isSCEVable(Ty) &&
1180 "This is not a conversion to a SCEVable type!");
1181 Ty = getEffectiveSCEVType(Ty);
1182
1183 // Sign-extend negative constants.
1184 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
1185 if (SC->getValue()->getValue().isNegative())
1186 return getSignExtendExpr(Op, Ty);
1187
1188 // Peel off a truncate cast.
1189 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00001190 const SCEV *NewOp = T->getOperand();
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00001191 if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
1192 return getAnyExtendExpr(NewOp, Ty);
1193 return getTruncateOrNoop(NewOp, Ty);
1194 }
1195
1196 // Next try a zext cast. If the cast is folded, use it.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001197 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00001198 if (!isa<SCEVZeroExtendExpr>(ZExt))
1199 return ZExt;
1200
1201 // Next try a sext cast. If the cast is folded, use it.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001202 const SCEV *SExt = getSignExtendExpr(Op, Ty);
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00001203 if (!isa<SCEVSignExtendExpr>(SExt))
1204 return SExt;
1205
Dan Gohmana10756e2010-01-21 02:09:26 +00001206 // Force the cast to be folded into the operands of an addrec.
1207 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
1208 SmallVector<const SCEV *, 4> Ops;
1209 for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end();
1210 I != E; ++I)
1211 Ops.push_back(getAnyExtendExpr(*I, Ty));
1212 return getAddRecExpr(Ops, AR->getLoop());
1213 }
1214
Dan Gohmanf53462d2010-07-15 20:02:11 +00001215 // As a special case, fold anyext(undef) to undef. We don't want to
1216 // know too much about SCEVUnknowns, but this special case is handy
1217 // and harmless.
1218 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Op))
1219 if (isa<UndefValue>(U->getValue()))
1220 return getSCEV(UndefValue::get(Ty));
1221
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00001222 // If the expression is obviously signed, use the sext cast value.
1223 if (isa<SCEVSMaxExpr>(Op))
1224 return SExt;
1225
1226 // Absent any other information, use the zext cast value.
1227 return ZExt;
1228}
1229
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001230/// CollectAddOperandsWithScales - Process the given Ops list, which is
1231/// a list of operands to be added under the given scale, update the given
1232/// map. This is a helper function for getAddRecExpr. As an example of
1233/// what it does, given a sequence of operands that would form an add
1234/// expression like this:
1235///
1236/// m + n + 13 + (A * (o + p + (B * q + m + 29))) + r + (-1 * r)
1237///
1238/// where A and B are constants, update the map with these values:
1239///
1240/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
1241///
1242/// and add 13 + A*B*29 to AccumulatedConstant.
1243/// This will allow getAddRecExpr to produce this:
1244///
1245/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
1246///
1247/// This form often exposes folding opportunities that are hidden in
1248/// the original operand list.
1249///
1250/// Return true iff it appears that any interesting folding opportunities
1251/// may be exposed. This helps getAddRecExpr short-circuit extra work in
1252/// the common case where no interesting opportunities are present, and
1253/// is also used as a check to avoid infinite recursion.
1254///
1255static bool
Dan Gohman0bba49c2009-07-07 17:06:11 +00001256CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M,
1257 SmallVector<const SCEV *, 8> &NewOps,
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001258 APInt &AccumulatedConstant,
Dan Gohmanf9e64722010-03-18 01:17:13 +00001259 const SCEV *const *Ops, size_t NumOperands,
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001260 const APInt &Scale,
1261 ScalarEvolution &SE) {
1262 bool Interesting = false;
1263
Dan Gohmane0f0c7b2010-06-18 19:12:32 +00001264 // Iterate over the add operands. They are sorted, with constants first.
1265 unsigned i = 0;
1266 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
1267 ++i;
1268 // Pull a buried constant out to the outside.
1269 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
1270 Interesting = true;
1271 AccumulatedConstant += Scale * C->getValue()->getValue();
1272 }
1273
1274 // Next comes everything else. We're especially interested in multiplies
1275 // here, but they're in the middle, so just visit the rest with one loop.
1276 for (; i != NumOperands; ++i) {
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001277 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]);
1278 if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
1279 APInt NewScale =
1280 Scale * cast<SCEVConstant>(Mul->getOperand(0))->getValue()->getValue();
1281 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
1282 // A multiplication of a constant with another add; recurse.
Dan Gohmanf9e64722010-03-18 01:17:13 +00001283 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001284 Interesting |=
1285 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
Dan Gohmanf9e64722010-03-18 01:17:13 +00001286 Add->op_begin(), Add->getNumOperands(),
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001287 NewScale, SE);
1288 } else {
1289 // A multiplication of a constant with some other value. Update
1290 // the map.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001291 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end());
1292 const SCEV *Key = SE.getMulExpr(MulOps);
1293 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
Dan Gohman23737e02009-06-29 18:25:52 +00001294 M.insert(std::make_pair(Key, NewScale));
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001295 if (Pair.second) {
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001296 NewOps.push_back(Pair.first->first);
1297 } else {
1298 Pair.first->second += NewScale;
1299 // The map already had an entry for this value, which may indicate
1300 // a folding opportunity.
1301 Interesting = true;
1302 }
1303 }
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001304 } else {
1305 // An ordinary operand. Update the map.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001306 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
Dan Gohman23737e02009-06-29 18:25:52 +00001307 M.insert(std::make_pair(Ops[i], Scale));
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001308 if (Pair.second) {
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001309 NewOps.push_back(Pair.first->first);
1310 } else {
1311 Pair.first->second += Scale;
1312 // The map already had an entry for this value, which may indicate
1313 // a folding opportunity.
1314 Interesting = true;
1315 }
1316 }
1317 }
1318
1319 return Interesting;
1320}
1321
1322namespace {
1323 struct APIntCompare {
1324 bool operator()(const APInt &LHS, const APInt &RHS) const {
1325 return LHS.ult(RHS);
1326 }
1327 };
1328}
1329
Dan Gohman6c0866c2009-05-24 23:45:28 +00001330/// getAddExpr - Get a canonical add expression, or something simpler if
1331/// possible.
Dan Gohman3645b012009-10-09 00:10:36 +00001332const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
1333 bool HasNUW, bool HasNSW) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001334 assert(!Ops.empty() && "Cannot get empty add!");
Chris Lattner627018b2004-04-07 16:16:11 +00001335 if (Ops.size() == 1) return Ops[0];
Dan Gohmanf78a9782009-05-18 15:44:58 +00001336#ifndef NDEBUG
Dan Gohmanc72f0c82010-06-18 19:09:27 +00001337 const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
Dan Gohmanf78a9782009-05-18 15:44:58 +00001338 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
Dan Gohmanc72f0c82010-06-18 19:09:27 +00001339 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
Dan Gohmanf78a9782009-05-18 15:44:58 +00001340 "SCEVAddExpr operand types don't match!");
1341#endif
Chris Lattner53e677a2004-04-02 20:23:17 +00001342
Dan Gohmana10756e2010-01-21 02:09:26 +00001343 // If HasNSW is true and all the operands are non-negative, infer HasNUW.
1344 if (!HasNUW && HasNSW) {
1345 bool All = true;
Dan Gohman2d16fc52010-08-16 16:27:53 +00001346 for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(),
1347 E = Ops.end(); I != E; ++I)
1348 if (!isKnownNonNegative(*I)) {
Dan Gohmana10756e2010-01-21 02:09:26 +00001349 All = false;
1350 break;
1351 }
1352 if (All) HasNUW = true;
1353 }
1354
Chris Lattner53e677a2004-04-02 20:23:17 +00001355 // Sort by complexity, this groups all similar expression types together.
Dan Gohman72861302009-05-07 14:39:04 +00001356 GroupByComplexity(Ops, LI);
Chris Lattner53e677a2004-04-02 20:23:17 +00001357
1358 // If there are any constants, fold them together.
1359 unsigned Idx = 0;
Dan Gohman622ed672009-05-04 22:02:23 +00001360 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001361 ++Idx;
Chris Lattner627018b2004-04-07 16:16:11 +00001362 assert(Idx < Ops.size());
Dan Gohman622ed672009-05-04 22:02:23 +00001363 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001364 // We found two constants, fold them together!
Dan Gohmana82752c2009-06-14 22:47:23 +00001365 Ops[0] = getConstant(LHSC->getValue()->getValue() +
1366 RHSC->getValue()->getValue());
Dan Gohman7f7c4362009-06-14 22:53:57 +00001367 if (Ops.size() == 2) return Ops[0];
Nick Lewycky3e630762008-02-20 06:48:22 +00001368 Ops.erase(Ops.begin()+1); // Erase the folded element
Nick Lewycky3e630762008-02-20 06:48:22 +00001369 LHSC = cast<SCEVConstant>(Ops[0]);
Chris Lattner53e677a2004-04-02 20:23:17 +00001370 }
1371
1372 // If we are left with a constant zero being added, strip it off.
Dan Gohmanbca091d2010-04-12 23:08:18 +00001373 if (LHSC->getValue()->isZero()) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001374 Ops.erase(Ops.begin());
1375 --Idx;
1376 }
Chris Lattner53e677a2004-04-02 20:23:17 +00001377
Dan Gohmanbca091d2010-04-12 23:08:18 +00001378 if (Ops.size() == 1) return Ops[0];
1379 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00001380
Chris Lattner53e677a2004-04-02 20:23:17 +00001381 // Okay, check to see if the same value occurs in the operand list twice. If
1382 // so, merge them together into an multiply expression. Since we sorted the
1383 // list, these values are required to be adjacent.
1384 const Type *Ty = Ops[0]->getType();
Dan Gohmandc7692b2010-08-12 14:46:54 +00001385 bool FoundMatch = false;
Chris Lattner53e677a2004-04-02 20:23:17 +00001386 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
1387 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
1388 // Found a match, merge the two values into a multiply, and add any
1389 // remaining values to the result.
Dan Gohmandeff6212010-05-03 22:09:21 +00001390 const SCEV *Two = getConstant(Ty, 2);
Dan Gohman58a85b92010-08-13 20:17:14 +00001391 const SCEV *Mul = getMulExpr(Two, Ops[i]);
Chris Lattner53e677a2004-04-02 20:23:17 +00001392 if (Ops.size() == 2)
1393 return Mul;
Dan Gohmandc7692b2010-08-12 14:46:54 +00001394 Ops[i] = Mul;
1395 Ops.erase(Ops.begin()+i+1);
1396 --i; --e;
1397 FoundMatch = true;
Chris Lattner53e677a2004-04-02 20:23:17 +00001398 }
Dan Gohmandc7692b2010-08-12 14:46:54 +00001399 if (FoundMatch)
1400 return getAddExpr(Ops, HasNUW, HasNSW);
Chris Lattner53e677a2004-04-02 20:23:17 +00001401
Dan Gohman728c7f32009-05-08 21:03:19 +00001402 // Check for truncates. If all the operands are truncated from the same
1403 // type, see if factoring out the truncate would permit the result to be
1404 // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n)
1405 // if the contents of the resulting outer trunc fold to something simple.
1406 for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) {
1407 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]);
1408 const Type *DstType = Trunc->getType();
1409 const Type *SrcType = Trunc->getOperand()->getType();
Dan Gohman0bba49c2009-07-07 17:06:11 +00001410 SmallVector<const SCEV *, 8> LargeOps;
Dan Gohman728c7f32009-05-08 21:03:19 +00001411 bool Ok = true;
1412 // Check all the operands to see if they can be represented in the
1413 // source type of the truncate.
1414 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
1415 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) {
1416 if (T->getOperand()->getType() != SrcType) {
1417 Ok = false;
1418 break;
1419 }
1420 LargeOps.push_back(T->getOperand());
1421 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
Dan Gohmanc6863982010-04-23 01:51:29 +00001422 LargeOps.push_back(getAnyExtendExpr(C, SrcType));
Dan Gohman728c7f32009-05-08 21:03:19 +00001423 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00001424 SmallVector<const SCEV *, 8> LargeMulOps;
Dan Gohman728c7f32009-05-08 21:03:19 +00001425 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
1426 if (const SCEVTruncateExpr *T =
1427 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
1428 if (T->getOperand()->getType() != SrcType) {
1429 Ok = false;
1430 break;
1431 }
1432 LargeMulOps.push_back(T->getOperand());
1433 } else if (const SCEVConstant *C =
1434 dyn_cast<SCEVConstant>(M->getOperand(j))) {
Dan Gohmanc6863982010-04-23 01:51:29 +00001435 LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
Dan Gohman728c7f32009-05-08 21:03:19 +00001436 } else {
1437 Ok = false;
1438 break;
1439 }
1440 }
1441 if (Ok)
1442 LargeOps.push_back(getMulExpr(LargeMulOps));
1443 } else {
1444 Ok = false;
1445 break;
1446 }
1447 }
1448 if (Ok) {
1449 // Evaluate the expression in the larger type.
Dan Gohman3645b012009-10-09 00:10:36 +00001450 const SCEV *Fold = getAddExpr(LargeOps, HasNUW, HasNSW);
Dan Gohman728c7f32009-05-08 21:03:19 +00001451 // If it folds to something simple, use it. Otherwise, don't.
1452 if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
1453 return getTruncateExpr(Fold, DstType);
1454 }
1455 }
1456
1457 // Skip past any other cast SCEVs.
Dan Gohmanf50cd742007-06-18 19:30:09 +00001458 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
1459 ++Idx;
1460
1461 // If there are add operands they would be next.
Chris Lattner53e677a2004-04-02 20:23:17 +00001462 if (Idx < Ops.size()) {
1463 bool DeletedAdd = false;
Dan Gohman622ed672009-05-04 22:02:23 +00001464 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001465 // If we have an add, expand the add operands onto the end of the operands
1466 // list.
Chris Lattner53e677a2004-04-02 20:23:17 +00001467 Ops.erase(Ops.begin()+Idx);
Dan Gohman403a8cd2010-06-21 19:47:52 +00001468 Ops.append(Add->op_begin(), Add->op_end());
Chris Lattner53e677a2004-04-02 20:23:17 +00001469 DeletedAdd = true;
1470 }
1471
1472 // If we deleted at least one add, we added operands to the end of the list,
1473 // and they are not necessarily sorted. Recurse to resort and resimplify
Dan Gohman3f46a3a2010-03-01 17:49:51 +00001474 // any operands we just acquired.
Chris Lattner53e677a2004-04-02 20:23:17 +00001475 if (DeletedAdd)
Dan Gohman246b2562007-10-22 18:31:58 +00001476 return getAddExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001477 }
1478
1479 // Skip over the add expression until we get to a multiply.
1480 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
1481 ++Idx;
1482
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001483 // Check to see if there are any folding opportunities present with
1484 // operands multiplied by constant values.
1485 if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
1486 uint64_t BitWidth = getTypeSizeInBits(Ty);
Dan Gohman0bba49c2009-07-07 17:06:11 +00001487 DenseMap<const SCEV *, APInt> M;
1488 SmallVector<const SCEV *, 8> NewOps;
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001489 APInt AccumulatedConstant(BitWidth, 0);
1490 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
Dan Gohmanf9e64722010-03-18 01:17:13 +00001491 Ops.data(), Ops.size(),
1492 APInt(BitWidth, 1), *this)) {
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001493 // Some interesting folding opportunity is present, so its worthwhile to
1494 // re-generate the operands list. Group the operands by constant scale,
1495 // to avoid multiplying by the same constant scale multiple times.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001496 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
Dan Gohman8d9c7a62010-08-16 16:30:01 +00001497 for (SmallVector<const SCEV *, 8>::const_iterator I = NewOps.begin(),
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001498 E = NewOps.end(); I != E; ++I)
1499 MulOpLists[M.find(*I)->second].push_back(*I);
1500 // Re-generate the operands list.
1501 Ops.clear();
1502 if (AccumulatedConstant != 0)
1503 Ops.push_back(getConstant(AccumulatedConstant));
Dan Gohman64a845e2009-06-24 04:48:43 +00001504 for (std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare>::iterator
1505 I = MulOpLists.begin(), E = MulOpLists.end(); I != E; ++I)
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001506 if (I->first != 0)
Dan Gohman64a845e2009-06-24 04:48:43 +00001507 Ops.push_back(getMulExpr(getConstant(I->first),
1508 getAddExpr(I->second)));
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001509 if (Ops.empty())
Dan Gohmandeff6212010-05-03 22:09:21 +00001510 return getConstant(Ty, 0);
Dan Gohmanbd59d7b2009-06-14 22:58:51 +00001511 if (Ops.size() == 1)
1512 return Ops[0];
1513 return getAddExpr(Ops);
1514 }
1515 }
1516
Chris Lattner53e677a2004-04-02 20:23:17 +00001517 // If we are adding something to a multiply expression, make sure the
1518 // something is not already an operand of the multiply. If so, merge it into
1519 // the multiply.
1520 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
Dan Gohman35738ac2009-05-04 22:30:44 +00001521 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
Chris Lattner53e677a2004-04-02 20:23:17 +00001522 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
Dan Gohman35738ac2009-05-04 22:30:44 +00001523 const SCEV *MulOpSCEV = Mul->getOperand(MulOp);
Dan Gohman918e76b2010-08-12 14:52:55 +00001524 if (isa<SCEVConstant>(MulOpSCEV))
1525 continue;
Chris Lattner53e677a2004-04-02 20:23:17 +00001526 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp)
Dan Gohman918e76b2010-08-12 14:52:55 +00001527 if (MulOpSCEV == Ops[AddOp]) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001528 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
Dan Gohman0bba49c2009-07-07 17:06:11 +00001529 const SCEV *InnerMul = Mul->getOperand(MulOp == 0);
Chris Lattner53e677a2004-04-02 20:23:17 +00001530 if (Mul->getNumOperands() != 2) {
1531 // If the multiply has more than two operands, we must get the
1532 // Y*Z term.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001533 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(), Mul->op_end());
Chris Lattner53e677a2004-04-02 20:23:17 +00001534 MulOps.erase(MulOps.begin()+MulOp);
Dan Gohman246b2562007-10-22 18:31:58 +00001535 InnerMul = getMulExpr(MulOps);
Chris Lattner53e677a2004-04-02 20:23:17 +00001536 }
Dan Gohmandeff6212010-05-03 22:09:21 +00001537 const SCEV *One = getConstant(Ty, 1);
Dan Gohman58a85b92010-08-13 20:17:14 +00001538 const SCEV *AddOne = getAddExpr(One, InnerMul);
Dan Gohman918e76b2010-08-12 14:52:55 +00001539 const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV);
Chris Lattner53e677a2004-04-02 20:23:17 +00001540 if (Ops.size() == 2) return OuterMul;
1541 if (AddOp < Idx) {
1542 Ops.erase(Ops.begin()+AddOp);
1543 Ops.erase(Ops.begin()+Idx-1);
1544 } else {
1545 Ops.erase(Ops.begin()+Idx);
1546 Ops.erase(Ops.begin()+AddOp-1);
1547 }
1548 Ops.push_back(OuterMul);
Dan Gohman246b2562007-10-22 18:31:58 +00001549 return getAddExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001550 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00001551
Chris Lattner53e677a2004-04-02 20:23:17 +00001552 // Check this multiply against other multiplies being added together.
Dan Gohman727356f2010-08-12 15:00:23 +00001553 bool AnyFold = false;
Chris Lattner53e677a2004-04-02 20:23:17 +00001554 for (unsigned OtherMulIdx = Idx+1;
1555 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]);
1556 ++OtherMulIdx) {
Dan Gohman35738ac2009-05-04 22:30:44 +00001557 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]);
Chris Lattner53e677a2004-04-02 20:23:17 +00001558 // If MulOp occurs in OtherMul, we can fold the two multiplies
1559 // together.
1560 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands();
1561 OMulOp != e; ++OMulOp)
1562 if (OtherMul->getOperand(OMulOp) == MulOpSCEV) {
1563 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E))
Dan Gohman0bba49c2009-07-07 17:06:11 +00001564 const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0);
Chris Lattner53e677a2004-04-02 20:23:17 +00001565 if (Mul->getNumOperands() != 2) {
Dan Gohman64a845e2009-06-24 04:48:43 +00001566 SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
1567 Mul->op_end());
Chris Lattner53e677a2004-04-02 20:23:17 +00001568 MulOps.erase(MulOps.begin()+MulOp);
Dan Gohman246b2562007-10-22 18:31:58 +00001569 InnerMul1 = getMulExpr(MulOps);
Chris Lattner53e677a2004-04-02 20:23:17 +00001570 }
Dan Gohman0bba49c2009-07-07 17:06:11 +00001571 const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0);
Chris Lattner53e677a2004-04-02 20:23:17 +00001572 if (OtherMul->getNumOperands() != 2) {
Dan Gohman64a845e2009-06-24 04:48:43 +00001573 SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(),
1574 OtherMul->op_end());
Chris Lattner53e677a2004-04-02 20:23:17 +00001575 MulOps.erase(MulOps.begin()+OMulOp);
Dan Gohman246b2562007-10-22 18:31:58 +00001576 InnerMul2 = getMulExpr(MulOps);
Chris Lattner53e677a2004-04-02 20:23:17 +00001577 }
Dan Gohman0bba49c2009-07-07 17:06:11 +00001578 const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2);
1579 const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum);
Chris Lattner53e677a2004-04-02 20:23:17 +00001580 if (Ops.size() == 2) return OuterMul;
Dan Gohman727356f2010-08-12 15:00:23 +00001581 Ops[Idx] = OuterMul;
1582 Ops.erase(Ops.begin()+OtherMulIdx);
1583 OtherMulIdx = Idx;
1584 AnyFold = true;
Chris Lattner53e677a2004-04-02 20:23:17 +00001585 }
1586 }
Dan Gohman727356f2010-08-12 15:00:23 +00001587 if (AnyFold)
1588 return getAddExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001589 }
1590 }
1591
1592 // If there are any add recurrences in the operands list, see if any other
1593 // added values are loop invariant. If so, we can fold them into the
1594 // recurrence.
1595 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
1596 ++Idx;
1597
1598 // Scan over all recurrences, trying to fold loop invariants into them.
1599 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
1600 // Scan all of the other operands to this add and add them to the vector if
1601 // they are loop invariant w.r.t. the recurrence.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001602 SmallVector<const SCEV *, 8> LIOps;
Dan Gohman35738ac2009-05-04 22:30:44 +00001603 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
Dan Gohmanbca091d2010-04-12 23:08:18 +00001604 const Loop *AddRecLoop = AddRec->getLoop();
Chris Lattner53e677a2004-04-02 20:23:17 +00001605 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
Dan Gohmanbca091d2010-04-12 23:08:18 +00001606 if (Ops[i]->isLoopInvariant(AddRecLoop)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001607 LIOps.push_back(Ops[i]);
1608 Ops.erase(Ops.begin()+i);
1609 --i; --e;
1610 }
1611
1612 // If we found some loop invariants, fold them into the recurrence.
1613 if (!LIOps.empty()) {
Dan Gohman8dae1382008-09-14 17:21:12 +00001614 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
Chris Lattner53e677a2004-04-02 20:23:17 +00001615 LIOps.push_back(AddRec->getStart());
1616
Dan Gohman0bba49c2009-07-07 17:06:11 +00001617 SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(),
Dan Gohman3a5d4092009-12-18 03:57:04 +00001618 AddRec->op_end());
Dan Gohman246b2562007-10-22 18:31:58 +00001619 AddRecOps[0] = getAddExpr(LIOps);
Chris Lattner53e677a2004-04-02 20:23:17 +00001620
Dan Gohmanb9f96512010-06-30 07:16:37 +00001621 // Build the new addrec. Propagate the NUW and NSW flags if both the
1622 // outer add and the inner addrec are guaranteed to have no overflow.
1623 const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop,
1624 HasNUW && AddRec->hasNoUnsignedWrap(),
1625 HasNSW && AddRec->hasNoSignedWrap());
Dan Gohman59de33e2009-12-18 18:45:31 +00001626
Chris Lattner53e677a2004-04-02 20:23:17 +00001627 // If all of the other operands were loop invariant, we are done.
1628 if (Ops.size() == 1) return NewRec;
1629
1630 // Otherwise, add the folded AddRec by the non-liv parts.
1631 for (unsigned i = 0;; ++i)
1632 if (Ops[i] == AddRec) {
1633 Ops[i] = NewRec;
1634 break;
1635 }
Dan Gohman246b2562007-10-22 18:31:58 +00001636 return getAddExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001637 }
1638
1639 // Okay, if there weren't any loop invariants to be folded, check to see if
1640 // there are multiple AddRec's with the same loop induction variable being
1641 // added together. If so, we can fold them.
1642 for (unsigned OtherIdx = Idx+1;
1643 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);++OtherIdx)
1644 if (OtherIdx != Idx) {
Dan Gohman35738ac2009-05-04 22:30:44 +00001645 const SCEVAddRecExpr *OtherAddRec = cast<SCEVAddRecExpr>(Ops[OtherIdx]);
Dan Gohmanbca091d2010-04-12 23:08:18 +00001646 if (AddRecLoop == OtherAddRec->getLoop()) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001647 // Other + {A,+,B} + {C,+,D} --> Other + {A+C,+,B+D}
Dan Gohman64a845e2009-06-24 04:48:43 +00001648 SmallVector<const SCEV *, 4> NewOps(AddRec->op_begin(),
1649 AddRec->op_end());
Chris Lattner53e677a2004-04-02 20:23:17 +00001650 for (unsigned i = 0, e = OtherAddRec->getNumOperands(); i != e; ++i) {
1651 if (i >= NewOps.size()) {
Dan Gohman403a8cd2010-06-21 19:47:52 +00001652 NewOps.append(OtherAddRec->op_begin()+i,
Chris Lattner53e677a2004-04-02 20:23:17 +00001653 OtherAddRec->op_end());
1654 break;
1655 }
Dan Gohman246b2562007-10-22 18:31:58 +00001656 NewOps[i] = getAddExpr(NewOps[i], OtherAddRec->getOperand(i));
Chris Lattner53e677a2004-04-02 20:23:17 +00001657 }
Dan Gohmanbca091d2010-04-12 23:08:18 +00001658 const SCEV *NewAddRec = getAddRecExpr(NewOps, AddRecLoop);
Chris Lattner53e677a2004-04-02 20:23:17 +00001659
1660 if (Ops.size() == 2) return NewAddRec;
1661
1662 Ops.erase(Ops.begin()+Idx);
1663 Ops.erase(Ops.begin()+OtherIdx-1);
1664 Ops.push_back(NewAddRec);
Dan Gohman246b2562007-10-22 18:31:58 +00001665 return getAddExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001666 }
1667 }
1668
1669 // Otherwise couldn't fold anything into this recurrence. Move onto the
1670 // next one.
1671 }
1672
1673 // Okay, it looks like we really DO need an add expr. Check to see if we
1674 // already have one, otherwise create a new one.
Dan Gohman1c343752009-06-27 21:21:31 +00001675 FoldingSetNodeID ID;
1676 ID.AddInteger(scAddExpr);
1677 ID.AddInteger(Ops.size());
1678 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
1679 ID.AddPointer(Ops[i]);
1680 void *IP = 0;
Dan Gohmana10756e2010-01-21 02:09:26 +00001681 SCEVAddExpr *S =
1682 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
1683 if (!S) {
Dan Gohmanf9e64722010-03-18 01:17:13 +00001684 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
1685 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
Dan Gohman95531882010-03-18 18:49:47 +00001686 S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator),
1687 O, Ops.size());
Dan Gohmana10756e2010-01-21 02:09:26 +00001688 UniqueSCEVs.InsertNode(S, IP);
1689 }
Dan Gohman3645b012009-10-09 00:10:36 +00001690 if (HasNUW) S->setHasNoUnsignedWrap(true);
1691 if (HasNSW) S->setHasNoSignedWrap(true);
Dan Gohman1c343752009-06-27 21:21:31 +00001692 return S;
Chris Lattner53e677a2004-04-02 20:23:17 +00001693}
1694
Dan Gohman6c0866c2009-05-24 23:45:28 +00001695/// getMulExpr - Get a canonical multiply expression, or something simpler if
1696/// possible.
Dan Gohman3645b012009-10-09 00:10:36 +00001697const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
1698 bool HasNUW, bool HasNSW) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001699 assert(!Ops.empty() && "Cannot get empty mul!");
Dan Gohmana10756e2010-01-21 02:09:26 +00001700 if (Ops.size() == 1) return Ops[0];
Dan Gohmanf78a9782009-05-18 15:44:58 +00001701#ifndef NDEBUG
Dan Gohmanc4f77982010-08-16 16:13:54 +00001702 const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
Dan Gohmanf78a9782009-05-18 15:44:58 +00001703 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
Dan Gohmanc4f77982010-08-16 16:13:54 +00001704 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
Dan Gohmanf78a9782009-05-18 15:44:58 +00001705 "SCEVMulExpr operand types don't match!");
1706#endif
Chris Lattner53e677a2004-04-02 20:23:17 +00001707
Dan Gohmana10756e2010-01-21 02:09:26 +00001708 // If HasNSW is true and all the operands are non-negative, infer HasNUW.
1709 if (!HasNUW && HasNSW) {
1710 bool All = true;
Dan Gohman2d16fc52010-08-16 16:27:53 +00001711 for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(),
1712 E = Ops.end(); I != E; ++I)
1713 if (!isKnownNonNegative(*I)) {
Dan Gohmana10756e2010-01-21 02:09:26 +00001714 All = false;
1715 break;
1716 }
1717 if (All) HasNUW = true;
1718 }
1719
Chris Lattner53e677a2004-04-02 20:23:17 +00001720 // Sort by complexity, this groups all similar expression types together.
Dan Gohman72861302009-05-07 14:39:04 +00001721 GroupByComplexity(Ops, LI);
Chris Lattner53e677a2004-04-02 20:23:17 +00001722
1723 // If there are any constants, fold them together.
1724 unsigned Idx = 0;
Dan Gohman622ed672009-05-04 22:02:23 +00001725 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001726
1727 // C1*(C2+V) -> C1*C2 + C1*V
1728 if (Ops.size() == 2)
Dan Gohman622ed672009-05-04 22:02:23 +00001729 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1]))
Chris Lattner53e677a2004-04-02 20:23:17 +00001730 if (Add->getNumOperands() == 2 &&
1731 isa<SCEVConstant>(Add->getOperand(0)))
Dan Gohman246b2562007-10-22 18:31:58 +00001732 return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)),
1733 getMulExpr(LHSC, Add->getOperand(1)));
Chris Lattner53e677a2004-04-02 20:23:17 +00001734
Chris Lattner53e677a2004-04-02 20:23:17 +00001735 ++Idx;
Dan Gohman622ed672009-05-04 22:02:23 +00001736 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001737 // We found two constants, fold them together!
Owen Andersoneed707b2009-07-24 23:12:02 +00001738 ConstantInt *Fold = ConstantInt::get(getContext(),
1739 LHSC->getValue()->getValue() *
Nick Lewycky3e630762008-02-20 06:48:22 +00001740 RHSC->getValue()->getValue());
1741 Ops[0] = getConstant(Fold);
1742 Ops.erase(Ops.begin()+1); // Erase the folded element
1743 if (Ops.size() == 1) return Ops[0];
1744 LHSC = cast<SCEVConstant>(Ops[0]);
Chris Lattner53e677a2004-04-02 20:23:17 +00001745 }
1746
1747 // If we are left with a constant one being multiplied, strip it off.
1748 if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) {
1749 Ops.erase(Ops.begin());
1750 --Idx;
Reid Spencercae57542007-03-02 00:28:52 +00001751 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001752 // If we have a multiply of zero, it will always be zero.
1753 return Ops[0];
Dan Gohmana10756e2010-01-21 02:09:26 +00001754 } else if (Ops[0]->isAllOnesValue()) {
1755 // If we have a mul by -1 of an add, try distributing the -1 among the
1756 // add operands.
1757 if (Ops.size() == 2)
1758 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
1759 SmallVector<const SCEV *, 4> NewOps;
1760 bool AnyFolded = false;
1761 for (SCEVAddRecExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
1762 I != E; ++I) {
1763 const SCEV *Mul = getMulExpr(Ops[0], *I);
1764 if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
1765 NewOps.push_back(Mul);
1766 }
1767 if (AnyFolded)
1768 return getAddExpr(NewOps);
1769 }
Chris Lattner53e677a2004-04-02 20:23:17 +00001770 }
Dan Gohman3ab13122010-04-13 16:49:23 +00001771
1772 if (Ops.size() == 1)
1773 return Ops[0];
Chris Lattner53e677a2004-04-02 20:23:17 +00001774 }
1775
1776 // Skip over the add expression until we get to a multiply.
1777 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
1778 ++Idx;
1779
Chris Lattner53e677a2004-04-02 20:23:17 +00001780 // If there are mul operands inline them all into this expression.
1781 if (Idx < Ops.size()) {
1782 bool DeletedMul = false;
Dan Gohman622ed672009-05-04 22:02:23 +00001783 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001784 // If we have an mul, expand the mul operands onto the end of the operands
1785 // list.
Chris Lattner53e677a2004-04-02 20:23:17 +00001786 Ops.erase(Ops.begin()+Idx);
Dan Gohman403a8cd2010-06-21 19:47:52 +00001787 Ops.append(Mul->op_begin(), Mul->op_end());
Chris Lattner53e677a2004-04-02 20:23:17 +00001788 DeletedMul = true;
1789 }
1790
1791 // If we deleted at least one mul, we added operands to the end of the list,
1792 // and they are not necessarily sorted. Recurse to resort and resimplify
Dan Gohman3f46a3a2010-03-01 17:49:51 +00001793 // any operands we just acquired.
Chris Lattner53e677a2004-04-02 20:23:17 +00001794 if (DeletedMul)
Dan Gohman246b2562007-10-22 18:31:58 +00001795 return getMulExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001796 }
1797
1798 // If there are any add recurrences in the operands list, see if any other
1799 // added values are loop invariant. If so, we can fold them into the
1800 // recurrence.
1801 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
1802 ++Idx;
1803
1804 // Scan over all recurrences, trying to fold loop invariants into them.
1805 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
1806 // Scan all of the other operands to this mul and add them to the vector if
1807 // they are loop invariant w.r.t. the recurrence.
Dan Gohman0bba49c2009-07-07 17:06:11 +00001808 SmallVector<const SCEV *, 8> LIOps;
Dan Gohman35738ac2009-05-04 22:30:44 +00001809 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
Chris Lattner53e677a2004-04-02 20:23:17 +00001810 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
1811 if (Ops[i]->isLoopInvariant(AddRec->getLoop())) {
1812 LIOps.push_back(Ops[i]);
1813 Ops.erase(Ops.begin()+i);
1814 --i; --e;
1815 }
1816
1817 // If we found some loop invariants, fold them into the recurrence.
1818 if (!LIOps.empty()) {
Dan Gohman8dae1382008-09-14 17:21:12 +00001819 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
Dan Gohman0bba49c2009-07-07 17:06:11 +00001820 SmallVector<const SCEV *, 4> NewOps;
Chris Lattner53e677a2004-04-02 20:23:17 +00001821 NewOps.reserve(AddRec->getNumOperands());
Dan Gohman27ed6a42010-06-17 23:34:09 +00001822 const SCEV *Scale = getMulExpr(LIOps);
1823 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
1824 NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i)));
Chris Lattner53e677a2004-04-02 20:23:17 +00001825
Dan Gohmanb9f96512010-06-30 07:16:37 +00001826 // Build the new addrec. Propagate the NUW and NSW flags if both the
1827 // outer mul and the inner addrec are guaranteed to have no overflow.
Dan Gohmana10756e2010-01-21 02:09:26 +00001828 const SCEV *NewRec = getAddRecExpr(NewOps, AddRec->getLoop(),
1829 HasNUW && AddRec->hasNoUnsignedWrap(),
Dan Gohmanb9f96512010-06-30 07:16:37 +00001830 HasNSW && AddRec->hasNoSignedWrap());
Chris Lattner53e677a2004-04-02 20:23:17 +00001831
1832 // If all of the other operands were loop invariant, we are done.
1833 if (Ops.size() == 1) return NewRec;
1834
1835 // Otherwise, multiply the folded AddRec by the non-liv parts.
1836 for (unsigned i = 0;; ++i)
1837 if (Ops[i] == AddRec) {
1838 Ops[i] = NewRec;
1839 break;
1840 }
Dan Gohman246b2562007-10-22 18:31:58 +00001841 return getMulExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001842 }
1843
1844 // Okay, if there weren't any loop invariants to be folded, check to see if
1845 // there are multiple AddRec's with the same loop induction variable being
1846 // multiplied together. If so, we can fold them.
1847 for (unsigned OtherIdx = Idx+1;
1848 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);++OtherIdx)
1849 if (OtherIdx != Idx) {
Dan Gohman35738ac2009-05-04 22:30:44 +00001850 const SCEVAddRecExpr *OtherAddRec = cast<SCEVAddRecExpr>(Ops[OtherIdx]);
Chris Lattner53e677a2004-04-02 20:23:17 +00001851 if (AddRec->getLoop() == OtherAddRec->getLoop()) {
1852 // F * G --> {A,+,B} * {C,+,D} --> {A*C,+,F*D + G*B + B*D}
Dan Gohman35738ac2009-05-04 22:30:44 +00001853 const SCEVAddRecExpr *F = AddRec, *G = OtherAddRec;
Dan Gohman0c5e95d2010-08-16 16:34:09 +00001854 const SCEV *NewStart = getMulExpr(F->getStart(), G->getStart());
Dan Gohman0bba49c2009-07-07 17:06:11 +00001855 const SCEV *B = F->getStepRecurrence(*this);
1856 const SCEV *D = G->getStepRecurrence(*this);
1857 const SCEV *NewStep = getAddExpr(getMulExpr(F, D),
Dan Gohman0c5e95d2010-08-16 16:34:09 +00001858 getMulExpr(G, B),
1859 getMulExpr(B, D));
Dan Gohman0bba49c2009-07-07 17:06:11 +00001860 const SCEV *NewAddRec = getAddRecExpr(NewStart, NewStep,
Dan Gohman0c5e95d2010-08-16 16:34:09 +00001861 F->getLoop());
Chris Lattner53e677a2004-04-02 20:23:17 +00001862 if (Ops.size() == 2) return NewAddRec;
1863
1864 Ops.erase(Ops.begin()+Idx);
1865 Ops.erase(Ops.begin()+OtherIdx-1);
1866 Ops.push_back(NewAddRec);
Dan Gohman246b2562007-10-22 18:31:58 +00001867 return getMulExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00001868 }
1869 }
1870
1871 // Otherwise couldn't fold anything into this recurrence. Move onto the
1872 // next one.
1873 }
1874
1875 // Okay, it looks like we really DO need an mul expr. Check to see if we
1876 // already have one, otherwise create a new one.
Dan Gohman1c343752009-06-27 21:21:31 +00001877 FoldingSetNodeID ID;
1878 ID.AddInteger(scMulExpr);
1879 ID.AddInteger(Ops.size());
1880 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
1881 ID.AddPointer(Ops[i]);
1882 void *IP = 0;
Dan Gohmana10756e2010-01-21 02:09:26 +00001883 SCEVMulExpr *S =
1884 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
1885 if (!S) {
Dan Gohmanf9e64722010-03-18 01:17:13 +00001886 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
1887 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
Dan Gohman95531882010-03-18 18:49:47 +00001888 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
1889 O, Ops.size());
Dan Gohmana10756e2010-01-21 02:09:26 +00001890 UniqueSCEVs.InsertNode(S, IP);
1891 }
Dan Gohman3645b012009-10-09 00:10:36 +00001892 if (HasNUW) S->setHasNoUnsignedWrap(true);
1893 if (HasNSW) S->setHasNoSignedWrap(true);
Dan Gohman1c343752009-06-27 21:21:31 +00001894 return S;
Chris Lattner53e677a2004-04-02 20:23:17 +00001895}
1896
Andreas Bolka8a11c982009-08-07 22:55:26 +00001897/// getUDivExpr - Get a canonical unsigned division expression, or something
1898/// simpler if possible.
Dan Gohman9311ef62009-06-24 14:49:00 +00001899const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
1900 const SCEV *RHS) {
Dan Gohmanf78a9782009-05-18 15:44:58 +00001901 assert(getEffectiveSCEVType(LHS->getType()) ==
1902 getEffectiveSCEVType(RHS->getType()) &&
1903 "SCEVUDivExpr operand types don't match!");
1904
Dan Gohman622ed672009-05-04 22:02:23 +00001905 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00001906 if (RHSC->getValue()->equalsInt(1))
Dan Gohman4c0d5d52009-08-20 16:42:55 +00001907 return LHS; // X udiv 1 --> x
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00001908 // If the denominator is zero, the result of the udiv is undefined. Don't
1909 // try to analyze it, because the resolution chosen here may differ from
1910 // the resolution chosen in other parts of the compiler.
1911 if (!RHSC->getValue()->isZero()) {
1912 // Determine if the division can be folded into the operands of
1913 // its operands.
1914 // TODO: Generalize this to non-constants by using known-bits information.
1915 const Type *Ty = LHS->getType();
1916 unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros();
Dan Gohmanddd3a882010-08-04 19:52:50 +00001917 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00001918 // For non-power-of-two values, effectively round the value up to the
1919 // nearest power of two.
1920 if (!RHSC->getValue()->getValue().isPowerOf2())
1921 ++MaxShiftAmt;
1922 const IntegerType *ExtTy =
1923 IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
1924 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
1925 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
1926 if (const SCEVConstant *Step =
1927 dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this)))
1928 if (!Step->getValue()->getValue()
1929 .urem(RHSC->getValue()->getValue()) &&
1930 getZeroExtendExpr(AR, ExtTy) ==
1931 getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
1932 getZeroExtendExpr(Step, ExtTy),
1933 AR->getLoop())) {
1934 SmallVector<const SCEV *, 4> Operands;
1935 for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i)
1936 Operands.push_back(getUDivExpr(AR->getOperand(i), RHS));
1937 return getAddRecExpr(Operands, AR->getLoop());
Dan Gohman185cf032009-05-08 20:18:49 +00001938 }
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00001939 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
1940 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
1941 SmallVector<const SCEV *, 4> Operands;
1942 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i)
1943 Operands.push_back(getZeroExtendExpr(M->getOperand(i), ExtTy));
1944 if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands))
1945 // Find an operand that's safely divisible.
1946 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
1947 const SCEV *Op = M->getOperand(i);
1948 const SCEV *Div = getUDivExpr(Op, RHSC);
1949 if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
1950 Operands = SmallVector<const SCEV *, 4>(M->op_begin(),
1951 M->op_end());
1952 Operands[i] = Div;
1953 return getMulExpr(Operands);
1954 }
1955 }
Dan Gohman185cf032009-05-08 20:18:49 +00001956 }
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00001957 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
1958 if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(LHS)) {
1959 SmallVector<const SCEV *, 4> Operands;
1960 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i)
1961 Operands.push_back(getZeroExtendExpr(A->getOperand(i), ExtTy));
1962 if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
1963 Operands.clear();
1964 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
1965 const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
1966 if (isa<SCEVUDivExpr>(Op) ||
1967 getMulExpr(Op, RHS) != A->getOperand(i))
1968 break;
1969 Operands.push_back(Op);
1970 }
1971 if (Operands.size() == A->getNumOperands())
1972 return getAddExpr(Operands);
1973 }
1974 }
Dan Gohman185cf032009-05-08 20:18:49 +00001975
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00001976 // Fold if both operands are constant.
1977 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
1978 Constant *LHSCV = LHSC->getValue();
1979 Constant *RHSCV = RHSC->getValue();
1980 return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV,
1981 RHSCV)));
1982 }
Chris Lattner53e677a2004-04-02 20:23:17 +00001983 }
1984 }
1985
Dan Gohman1c343752009-06-27 21:21:31 +00001986 FoldingSetNodeID ID;
1987 ID.AddInteger(scUDivExpr);
1988 ID.AddPointer(LHS);
1989 ID.AddPointer(RHS);
1990 void *IP = 0;
1991 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
Dan Gohman95531882010-03-18 18:49:47 +00001992 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
1993 LHS, RHS);
Dan Gohman1c343752009-06-27 21:21:31 +00001994 UniqueSCEVs.InsertNode(S, IP);
1995 return S;
Chris Lattner53e677a2004-04-02 20:23:17 +00001996}
1997
1998
Dan Gohman6c0866c2009-05-24 23:45:28 +00001999/// getAddRecExpr - Get an add recurrence expression for the specified loop.
2000/// Simplify the expression as much as possible.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002001const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start,
Dan Gohman3645b012009-10-09 00:10:36 +00002002 const SCEV *Step, const Loop *L,
2003 bool HasNUW, bool HasNSW) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00002004 SmallVector<const SCEV *, 4> Operands;
Chris Lattner53e677a2004-04-02 20:23:17 +00002005 Operands.push_back(Start);
Dan Gohman622ed672009-05-04 22:02:23 +00002006 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
Chris Lattner53e677a2004-04-02 20:23:17 +00002007 if (StepChrec->getLoop() == L) {
Dan Gohman403a8cd2010-06-21 19:47:52 +00002008 Operands.append(StepChrec->op_begin(), StepChrec->op_end());
Dan Gohman246b2562007-10-22 18:31:58 +00002009 return getAddRecExpr(Operands, L);
Chris Lattner53e677a2004-04-02 20:23:17 +00002010 }
2011
2012 Operands.push_back(Step);
Dan Gohman3645b012009-10-09 00:10:36 +00002013 return getAddRecExpr(Operands, L, HasNUW, HasNSW);
Chris Lattner53e677a2004-04-02 20:23:17 +00002014}
2015
Dan Gohman6c0866c2009-05-24 23:45:28 +00002016/// getAddRecExpr - Get an add recurrence expression for the specified loop.
2017/// Simplify the expression as much as possible.
Dan Gohman64a845e2009-06-24 04:48:43 +00002018const SCEV *
Dan Gohman0bba49c2009-07-07 17:06:11 +00002019ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
Dan Gohman3645b012009-10-09 00:10:36 +00002020 const Loop *L,
2021 bool HasNUW, bool HasNSW) {
Chris Lattner53e677a2004-04-02 20:23:17 +00002022 if (Operands.size() == 1) return Operands[0];
Dan Gohmanf78a9782009-05-18 15:44:58 +00002023#ifndef NDEBUG
Dan Gohmanc4f77982010-08-16 16:13:54 +00002024 const Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
Dan Gohmanf78a9782009-05-18 15:44:58 +00002025 for (unsigned i = 1, e = Operands.size(); i != e; ++i)
Dan Gohmanc4f77982010-08-16 16:13:54 +00002026 assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy &&
Dan Gohmanf78a9782009-05-18 15:44:58 +00002027 "SCEVAddRecExpr operand types don't match!");
2028#endif
Chris Lattner53e677a2004-04-02 20:23:17 +00002029
Dan Gohmancfeb6a42008-06-18 16:23:07 +00002030 if (Operands.back()->isZero()) {
2031 Operands.pop_back();
Dan Gohman3645b012009-10-09 00:10:36 +00002032 return getAddRecExpr(Operands, L, HasNUW, HasNSW); // {X,+,0} --> X
Dan Gohmancfeb6a42008-06-18 16:23:07 +00002033 }
Chris Lattner53e677a2004-04-02 20:23:17 +00002034
Dan Gohmanbc028532010-02-19 18:49:22 +00002035 // It's tempting to want to call getMaxBackedgeTakenCount count here and
2036 // use that information to infer NUW and NSW flags. However, computing a
2037 // BE count requires calling getAddRecExpr, so we may not yet have a
2038 // meaningful BE count at this point (and if we don't, we'd be stuck
2039 // with a SCEVCouldNotCompute as the cached BE count).
2040
Dan Gohmana10756e2010-01-21 02:09:26 +00002041 // If HasNSW is true and all the operands are non-negative, infer HasNUW.
2042 if (!HasNUW && HasNSW) {
2043 bool All = true;
Dan Gohman2d16fc52010-08-16 16:27:53 +00002044 for (SmallVectorImpl<const SCEV *>::const_iterator I = Operands.begin(),
2045 E = Operands.end(); I != E; ++I)
2046 if (!isKnownNonNegative(*I)) {
Dan Gohmana10756e2010-01-21 02:09:26 +00002047 All = false;
2048 break;
2049 }
2050 if (All) HasNUW = true;
2051 }
2052
Dan Gohmand9cc7492008-08-08 18:33:12 +00002053 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
Dan Gohman622ed672009-05-04 22:02:23 +00002054 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
Dan Gohman5d984912009-12-18 01:14:11 +00002055 const Loop *NestedLoop = NestedAR->getLoop();
Dan Gohman9cba9782010-08-13 20:23:25 +00002056 if (L->contains(NestedLoop) ?
Dan Gohmana10756e2010-01-21 02:09:26 +00002057 (L->getLoopDepth() < NestedLoop->getLoopDepth()) :
Dan Gohman9cba9782010-08-13 20:23:25 +00002058 (!NestedLoop->contains(L) &&
Dan Gohmana10756e2010-01-21 02:09:26 +00002059 DT->dominates(L->getHeader(), NestedLoop->getHeader()))) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00002060 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(),
Dan Gohman5d984912009-12-18 01:14:11 +00002061 NestedAR->op_end());
Dan Gohmand9cc7492008-08-08 18:33:12 +00002062 Operands[0] = NestedAR->getStart();
Dan Gohman9a80b452009-06-26 22:36:20 +00002063 // AddRecs require their operands be loop-invariant with respect to their
2064 // loops. Don't perform this transformation if it would break this
2065 // requirement.
2066 bool AllInvariant = true;
2067 for (unsigned i = 0, e = Operands.size(); i != e; ++i)
2068 if (!Operands[i]->isLoopInvariant(L)) {
2069 AllInvariant = false;
2070 break;
2071 }
2072 if (AllInvariant) {
2073 NestedOperands[0] = getAddRecExpr(Operands, L);
2074 AllInvariant = true;
2075 for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i)
2076 if (!NestedOperands[i]->isLoopInvariant(NestedLoop)) {
2077 AllInvariant = false;
2078 break;
2079 }
2080 if (AllInvariant)
2081 // Ok, both add recurrences are valid after the transformation.
Dan Gohman3645b012009-10-09 00:10:36 +00002082 return getAddRecExpr(NestedOperands, NestedLoop, HasNUW, HasNSW);
Dan Gohman9a80b452009-06-26 22:36:20 +00002083 }
2084 // Reset Operands to its original state.
2085 Operands[0] = NestedAR;
Dan Gohmand9cc7492008-08-08 18:33:12 +00002086 }
2087 }
2088
Dan Gohman67847532010-01-19 22:27:22 +00002089 // Okay, it looks like we really DO need an addrec expr. Check to see if we
2090 // already have one, otherwise create a new one.
Dan Gohman1c343752009-06-27 21:21:31 +00002091 FoldingSetNodeID ID;
2092 ID.AddInteger(scAddRecExpr);
2093 ID.AddInteger(Operands.size());
2094 for (unsigned i = 0, e = Operands.size(); i != e; ++i)
2095 ID.AddPointer(Operands[i]);
2096 ID.AddPointer(L);
2097 void *IP = 0;
Dan Gohmana10756e2010-01-21 02:09:26 +00002098 SCEVAddRecExpr *S =
2099 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
2100 if (!S) {
Dan Gohmanf9e64722010-03-18 01:17:13 +00002101 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size());
2102 std::uninitialized_copy(Operands.begin(), Operands.end(), O);
Dan Gohman95531882010-03-18 18:49:47 +00002103 S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator),
2104 O, Operands.size(), L);
Dan Gohmana10756e2010-01-21 02:09:26 +00002105 UniqueSCEVs.InsertNode(S, IP);
2106 }
Dan Gohman3645b012009-10-09 00:10:36 +00002107 if (HasNUW) S->setHasNoUnsignedWrap(true);
2108 if (HasNSW) S->setHasNoSignedWrap(true);
Dan Gohman1c343752009-06-27 21:21:31 +00002109 return S;
Chris Lattner53e677a2004-04-02 20:23:17 +00002110}
2111
Dan Gohman9311ef62009-06-24 14:49:00 +00002112const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS,
2113 const SCEV *RHS) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00002114 SmallVector<const SCEV *, 2> Ops;
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002115 Ops.push_back(LHS);
2116 Ops.push_back(RHS);
2117 return getSMaxExpr(Ops);
2118}
2119
Dan Gohman0bba49c2009-07-07 17:06:11 +00002120const SCEV *
2121ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002122 assert(!Ops.empty() && "Cannot get empty smax!");
2123 if (Ops.size() == 1) return Ops[0];
Dan Gohmanf78a9782009-05-18 15:44:58 +00002124#ifndef NDEBUG
Dan Gohmanc4f77982010-08-16 16:13:54 +00002125 const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
Dan Gohmanf78a9782009-05-18 15:44:58 +00002126 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
Dan Gohmanc4f77982010-08-16 16:13:54 +00002127 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
Dan Gohmanf78a9782009-05-18 15:44:58 +00002128 "SCEVSMaxExpr operand types don't match!");
2129#endif
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002130
2131 // Sort by complexity, this groups all similar expression types together.
Dan Gohman72861302009-05-07 14:39:04 +00002132 GroupByComplexity(Ops, LI);
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002133
2134 // If there are any constants, fold them together.
2135 unsigned Idx = 0;
Dan Gohman622ed672009-05-04 22:02:23 +00002136 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002137 ++Idx;
2138 assert(Idx < Ops.size());
Dan Gohman622ed672009-05-04 22:02:23 +00002139 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002140 // We found two constants, fold them together!
Owen Andersoneed707b2009-07-24 23:12:02 +00002141 ConstantInt *Fold = ConstantInt::get(getContext(),
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002142 APIntOps::smax(LHSC->getValue()->getValue(),
2143 RHSC->getValue()->getValue()));
Nick Lewycky3e630762008-02-20 06:48:22 +00002144 Ops[0] = getConstant(Fold);
2145 Ops.erase(Ops.begin()+1); // Erase the folded element
2146 if (Ops.size() == 1) return Ops[0];
2147 LHSC = cast<SCEVConstant>(Ops[0]);
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002148 }
2149
Dan Gohmane5aceed2009-06-24 14:46:22 +00002150 // If we are left with a constant minimum-int, strip it off.
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002151 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) {
2152 Ops.erase(Ops.begin());
2153 --Idx;
Dan Gohmane5aceed2009-06-24 14:46:22 +00002154 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) {
2155 // If we have an smax with a constant maximum-int, it will always be
2156 // maximum-int.
2157 return Ops[0];
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002158 }
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002159
Dan Gohman3ab13122010-04-13 16:49:23 +00002160 if (Ops.size() == 1) return Ops[0];
2161 }
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002162
2163 // Find the first SMax
2164 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr)
2165 ++Idx;
2166
2167 // Check to see if one of the operands is an SMax. If so, expand its operands
2168 // onto our operand list, and recurse to simplify.
2169 if (Idx < Ops.size()) {
2170 bool DeletedSMax = false;
Dan Gohman622ed672009-05-04 22:02:23 +00002171 while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) {
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002172 Ops.erase(Ops.begin()+Idx);
Dan Gohman403a8cd2010-06-21 19:47:52 +00002173 Ops.append(SMax->op_begin(), SMax->op_end());
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002174 DeletedSMax = true;
2175 }
2176
2177 if (DeletedSMax)
2178 return getSMaxExpr(Ops);
2179 }
2180
2181 // Okay, check to see if the same value occurs in the operand list twice. If
2182 // so, delete one. Since we sorted the list, these values are required to
2183 // be adjacent.
2184 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
Dan Gohman28287792010-04-13 16:51:03 +00002185 // X smax Y smax Y --> X smax Y
2186 // X smax Y --> X, if X is always greater than Y
2187 if (Ops[i] == Ops[i+1] ||
2188 isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) {
2189 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
2190 --i; --e;
2191 } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) {
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002192 Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
2193 --i; --e;
2194 }
2195
2196 if (Ops.size() == 1) return Ops[0];
2197
2198 assert(!Ops.empty() && "Reduced smax down to nothing!");
2199
Nick Lewycky3e630762008-02-20 06:48:22 +00002200 // Okay, it looks like we really DO need an smax expr. Check to see if we
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002201 // already have one, otherwise create a new one.
Dan Gohman1c343752009-06-27 21:21:31 +00002202 FoldingSetNodeID ID;
2203 ID.AddInteger(scSMaxExpr);
2204 ID.AddInteger(Ops.size());
2205 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2206 ID.AddPointer(Ops[i]);
2207 void *IP = 0;
2208 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
Dan Gohmanf9e64722010-03-18 01:17:13 +00002209 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
2210 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
Dan Gohman95531882010-03-18 18:49:47 +00002211 SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator),
2212 O, Ops.size());
Dan Gohman1c343752009-06-27 21:21:31 +00002213 UniqueSCEVs.InsertNode(S, IP);
2214 return S;
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002215}
2216
Dan Gohman9311ef62009-06-24 14:49:00 +00002217const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS,
2218 const SCEV *RHS) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00002219 SmallVector<const SCEV *, 2> Ops;
Nick Lewycky3e630762008-02-20 06:48:22 +00002220 Ops.push_back(LHS);
2221 Ops.push_back(RHS);
2222 return getUMaxExpr(Ops);
2223}
2224
Dan Gohman0bba49c2009-07-07 17:06:11 +00002225const SCEV *
2226ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
Nick Lewycky3e630762008-02-20 06:48:22 +00002227 assert(!Ops.empty() && "Cannot get empty umax!");
2228 if (Ops.size() == 1) return Ops[0];
Dan Gohmanf78a9782009-05-18 15:44:58 +00002229#ifndef NDEBUG
Dan Gohmanc4f77982010-08-16 16:13:54 +00002230 const Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
Dan Gohmanf78a9782009-05-18 15:44:58 +00002231 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
Dan Gohmanc4f77982010-08-16 16:13:54 +00002232 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
Dan Gohmanf78a9782009-05-18 15:44:58 +00002233 "SCEVUMaxExpr operand types don't match!");
2234#endif
Nick Lewycky3e630762008-02-20 06:48:22 +00002235
2236 // Sort by complexity, this groups all similar expression types together.
Dan Gohman72861302009-05-07 14:39:04 +00002237 GroupByComplexity(Ops, LI);
Nick Lewycky3e630762008-02-20 06:48:22 +00002238
2239 // If there are any constants, fold them together.
2240 unsigned Idx = 0;
Dan Gohman622ed672009-05-04 22:02:23 +00002241 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
Nick Lewycky3e630762008-02-20 06:48:22 +00002242 ++Idx;
2243 assert(Idx < Ops.size());
Dan Gohman622ed672009-05-04 22:02:23 +00002244 while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
Nick Lewycky3e630762008-02-20 06:48:22 +00002245 // We found two constants, fold them together!
Owen Andersoneed707b2009-07-24 23:12:02 +00002246 ConstantInt *Fold = ConstantInt::get(getContext(),
Nick Lewycky3e630762008-02-20 06:48:22 +00002247 APIntOps::umax(LHSC->getValue()->getValue(),
2248 RHSC->getValue()->getValue()));
2249 Ops[0] = getConstant(Fold);
2250 Ops.erase(Ops.begin()+1); // Erase the folded element
2251 if (Ops.size() == 1) return Ops[0];
2252 LHSC = cast<SCEVConstant>(Ops[0]);
2253 }
2254
Dan Gohmane5aceed2009-06-24 14:46:22 +00002255 // If we are left with a constant minimum-int, strip it off.
Nick Lewycky3e630762008-02-20 06:48:22 +00002256 if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) {
2257 Ops.erase(Ops.begin());
2258 --Idx;
Dan Gohmane5aceed2009-06-24 14:46:22 +00002259 } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) {
2260 // If we have an umax with a constant maximum-int, it will always be
2261 // maximum-int.
2262 return Ops[0];
Nick Lewycky3e630762008-02-20 06:48:22 +00002263 }
Nick Lewycky3e630762008-02-20 06:48:22 +00002264
Dan Gohman3ab13122010-04-13 16:49:23 +00002265 if (Ops.size() == 1) return Ops[0];
2266 }
Nick Lewycky3e630762008-02-20 06:48:22 +00002267
2268 // Find the first UMax
2269 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr)
2270 ++Idx;
2271
2272 // Check to see if one of the operands is a UMax. If so, expand its operands
2273 // onto our operand list, and recurse to simplify.
2274 if (Idx < Ops.size()) {
2275 bool DeletedUMax = false;
Dan Gohman622ed672009-05-04 22:02:23 +00002276 while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) {
Nick Lewycky3e630762008-02-20 06:48:22 +00002277 Ops.erase(Ops.begin()+Idx);
Dan Gohman403a8cd2010-06-21 19:47:52 +00002278 Ops.append(UMax->op_begin(), UMax->op_end());
Nick Lewycky3e630762008-02-20 06:48:22 +00002279 DeletedUMax = true;
2280 }
2281
2282 if (DeletedUMax)
2283 return getUMaxExpr(Ops);
2284 }
2285
2286 // Okay, check to see if the same value occurs in the operand list twice. If
2287 // so, delete one. Since we sorted the list, these values are required to
2288 // be adjacent.
2289 for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
Dan Gohman28287792010-04-13 16:51:03 +00002290 // X umax Y umax Y --> X umax Y
2291 // X umax Y --> X, if X is always greater than Y
2292 if (Ops[i] == Ops[i+1] ||
2293 isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) {
2294 Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
2295 --i; --e;
2296 } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) {
Nick Lewycky3e630762008-02-20 06:48:22 +00002297 Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
2298 --i; --e;
2299 }
2300
2301 if (Ops.size() == 1) return Ops[0];
2302
2303 assert(!Ops.empty() && "Reduced umax down to nothing!");
2304
2305 // Okay, it looks like we really DO need a umax expr. Check to see if we
2306 // already have one, otherwise create a new one.
Dan Gohman1c343752009-06-27 21:21:31 +00002307 FoldingSetNodeID ID;
2308 ID.AddInteger(scUMaxExpr);
2309 ID.AddInteger(Ops.size());
2310 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2311 ID.AddPointer(Ops[i]);
2312 void *IP = 0;
2313 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
Dan Gohmanf9e64722010-03-18 01:17:13 +00002314 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
2315 std::uninitialized_copy(Ops.begin(), Ops.end(), O);
Dan Gohman95531882010-03-18 18:49:47 +00002316 SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator),
2317 O, Ops.size());
Dan Gohman1c343752009-06-27 21:21:31 +00002318 UniqueSCEVs.InsertNode(S, IP);
2319 return S;
Nick Lewycky3e630762008-02-20 06:48:22 +00002320}
2321
Dan Gohman9311ef62009-06-24 14:49:00 +00002322const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS,
2323 const SCEV *RHS) {
Dan Gohmanf9a9a992009-06-22 03:18:45 +00002324 // ~smax(~x, ~y) == smin(x, y).
2325 return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
2326}
2327
Dan Gohman9311ef62009-06-24 14:49:00 +00002328const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS,
2329 const SCEV *RHS) {
Dan Gohmanf9a9a992009-06-22 03:18:45 +00002330 // ~umax(~x, ~y) == umin(x, y)
2331 return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
2332}
2333
Dan Gohman4f8eea82010-02-01 18:27:38 +00002334const SCEV *ScalarEvolution::getSizeOfExpr(const Type *AllocTy) {
Dan Gohman6ab10f62010-04-12 23:03:26 +00002335 // If we have TargetData, we can bypass creating a target-independent
2336 // constant expression and then folding it back into a ConstantInt.
2337 // This is just a compile-time optimization.
2338 if (TD)
2339 return getConstant(TD->getIntPtrType(getContext()),
2340 TD->getTypeAllocSize(AllocTy));
2341
Dan Gohman4f8eea82010-02-01 18:27:38 +00002342 Constant *C = ConstantExpr::getSizeOf(AllocTy);
2343 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
Dan Gohman70001222010-05-28 16:12:08 +00002344 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
2345 C = Folded;
Dan Gohman4f8eea82010-02-01 18:27:38 +00002346 const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
2347 return getTruncateOrZeroExtend(getSCEV(C), Ty);
2348}
2349
2350const SCEV *ScalarEvolution::getAlignOfExpr(const Type *AllocTy) {
2351 Constant *C = ConstantExpr::getAlignOf(AllocTy);
2352 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
Dan Gohman70001222010-05-28 16:12:08 +00002353 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
2354 C = Folded;
Dan Gohman4f8eea82010-02-01 18:27:38 +00002355 const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
2356 return getTruncateOrZeroExtend(getSCEV(C), Ty);
2357}
2358
2359const SCEV *ScalarEvolution::getOffsetOfExpr(const StructType *STy,
2360 unsigned FieldNo) {
Dan Gohman6ab10f62010-04-12 23:03:26 +00002361 // If we have TargetData, we can bypass creating a target-independent
2362 // constant expression and then folding it back into a ConstantInt.
2363 // This is just a compile-time optimization.
2364 if (TD)
2365 return getConstant(TD->getIntPtrType(getContext()),
2366 TD->getStructLayout(STy)->getElementOffset(FieldNo));
2367
Dan Gohman0f5efe52010-01-28 02:15:55 +00002368 Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo);
2369 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
Dan Gohman70001222010-05-28 16:12:08 +00002370 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
2371 C = Folded;
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002372 const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy));
Dan Gohman0f5efe52010-01-28 02:15:55 +00002373 return getTruncateOrZeroExtend(getSCEV(C), Ty);
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002374}
2375
Dan Gohman4f8eea82010-02-01 18:27:38 +00002376const SCEV *ScalarEvolution::getOffsetOfExpr(const Type *CTy,
2377 Constant *FieldNo) {
2378 Constant *C = ConstantExpr::getOffsetOf(CTy, FieldNo);
Dan Gohman0f5efe52010-01-28 02:15:55 +00002379 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
Dan Gohman70001222010-05-28 16:12:08 +00002380 if (Constant *Folded = ConstantFoldConstantExpression(CE, TD))
2381 C = Folded;
Dan Gohman4f8eea82010-02-01 18:27:38 +00002382 const Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(CTy));
Dan Gohman0f5efe52010-01-28 02:15:55 +00002383 return getTruncateOrZeroExtend(getSCEV(C), Ty);
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002384}
2385
Dan Gohman0bba49c2009-07-07 17:06:11 +00002386const SCEV *ScalarEvolution::getUnknown(Value *V) {
Dan Gohman6bbcba12009-06-24 00:54:57 +00002387 // Don't attempt to do anything other than create a SCEVUnknown object
2388 // here. createSCEV only calls getUnknown after checking for all other
2389 // interesting possibilities, and any other code that calls getUnknown
2390 // is doing so in order to hide a value from SCEV canonicalization.
2391
Dan Gohman1c343752009-06-27 21:21:31 +00002392 FoldingSetNodeID ID;
2393 ID.AddInteger(scUnknown);
2394 ID.AddPointer(V);
2395 void *IP = 0;
Dan Gohmanab37f502010-08-02 23:49:30 +00002396 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
2397 assert(cast<SCEVUnknown>(S)->getValue() == V &&
2398 "Stale SCEVUnknown in uniquing map!");
2399 return S;
2400 }
2401 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
2402 FirstUnknown);
2403 FirstUnknown = cast<SCEVUnknown>(S);
Dan Gohman1c343752009-06-27 21:21:31 +00002404 UniqueSCEVs.InsertNode(S, IP);
2405 return S;
Chris Lattner0a7f98c2004-04-15 15:07:24 +00002406}
2407
Chris Lattner53e677a2004-04-02 20:23:17 +00002408//===----------------------------------------------------------------------===//
Chris Lattner53e677a2004-04-02 20:23:17 +00002409// Basic SCEV Analysis and PHI Idiom Recognition Code
2410//
2411
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002412/// isSCEVable - Test if values of the given type are analyzable within
2413/// the SCEV framework. This primarily includes integer types, and it
2414/// can optionally include pointer types if the ScalarEvolution class
2415/// has access to target-specific information.
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002416bool ScalarEvolution::isSCEVable(const Type *Ty) const {
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002417 // Integers and pointers are always SCEVable.
Duncan Sands1df98592010-02-16 11:11:14 +00002418 return Ty->isIntegerTy() || Ty->isPointerTy();
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002419}
2420
2421/// getTypeSizeInBits - Return the size in bits of the specified type,
2422/// for which isSCEVable must return true.
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002423uint64_t ScalarEvolution::getTypeSizeInBits(const Type *Ty) const {
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002424 assert(isSCEVable(Ty) && "Type is not SCEVable!");
2425
2426 // If we have a TargetData, use it!
2427 if (TD)
2428 return TD->getTypeSizeInBits(Ty);
2429
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002430 // Integer types have fixed sizes.
Duncan Sandsb0bc6c32010-02-15 16:12:20 +00002431 if (Ty->isIntegerTy())
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002432 return Ty->getPrimitiveSizeInBits();
2433
2434 // The only other support type is pointer. Without TargetData, conservatively
2435 // assume pointers are 64-bit.
Duncan Sands1df98592010-02-16 11:11:14 +00002436 assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!");
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002437 return 64;
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002438}
2439
2440/// getEffectiveSCEVType - Return a type with the same bitwidth as
2441/// the given type and which represents how SCEV will treat the given
2442/// type, for which isSCEVable must return true. For pointer types,
2443/// this is the pointer-sized integer type.
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002444const Type *ScalarEvolution::getEffectiveSCEVType(const Type *Ty) const {
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002445 assert(isSCEVable(Ty) && "Type is not SCEVable!");
2446
Duncan Sandsb0bc6c32010-02-15 16:12:20 +00002447 if (Ty->isIntegerTy())
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002448 return Ty;
2449
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002450 // The only other support type is pointer.
Duncan Sands1df98592010-02-16 11:11:14 +00002451 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002452 if (TD) return TD->getIntPtrType(getContext());
2453
2454 // Without TargetData, conservatively assume pointers are 64-bit.
2455 return Type::getInt64Ty(getContext());
Dan Gohman2d1be872009-04-16 03:18:22 +00002456}
Chris Lattner53e677a2004-04-02 20:23:17 +00002457
Dan Gohman0bba49c2009-07-07 17:06:11 +00002458const SCEV *ScalarEvolution::getCouldNotCompute() {
Dan Gohman1c343752009-06-27 21:21:31 +00002459 return &CouldNotCompute;
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00002460}
2461
Chris Lattner53e677a2004-04-02 20:23:17 +00002462/// getSCEV - Return an existing SCEV if it exists, otherwise analyze the
2463/// expression and create a new one.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002464const SCEV *ScalarEvolution::getSCEV(Value *V) {
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002465 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
Chris Lattner53e677a2004-04-02 20:23:17 +00002466
Dan Gohman8d9c7a62010-08-16 16:30:01 +00002467 std::map<SCEVCallbackVH, const SCEV *>::const_iterator I = Scalars.find(V);
Chris Lattner53e677a2004-04-02 20:23:17 +00002468 if (I != Scalars.end()) return I->second;
Dan Gohman0bba49c2009-07-07 17:06:11 +00002469 const SCEV *S = createSCEV(V);
Dan Gohman619d3322010-08-16 16:31:39 +00002470
2471 // The process of creating a SCEV for V may have caused other SCEVs
2472 // to have been created, so it's necessary to insert the new entry
2473 // from scratch, rather than trying to remember the insert position
2474 // above.
Dan Gohman35738ac2009-05-04 22:30:44 +00002475 Scalars.insert(std::make_pair(SCEVCallbackVH(V, this), S));
Chris Lattner53e677a2004-04-02 20:23:17 +00002476 return S;
2477}
2478
Dan Gohman2d1be872009-04-16 03:18:22 +00002479/// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
2480///
Dan Gohman0bba49c2009-07-07 17:06:11 +00002481const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) {
Dan Gohman622ed672009-05-04 22:02:23 +00002482 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
Owen Anderson0a5372e2009-07-13 04:09:18 +00002483 return getConstant(
Owen Andersonbaf3c402009-07-29 18:55:55 +00002484 cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
Dan Gohman2d1be872009-04-16 03:18:22 +00002485
2486 const Type *Ty = V->getType();
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002487 Ty = getEffectiveSCEVType(Ty);
Owen Anderson73c6b712009-07-13 20:58:05 +00002488 return getMulExpr(V,
Owen Andersona7235ea2009-07-31 20:28:14 +00002489 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))));
Dan Gohman2d1be872009-04-16 03:18:22 +00002490}
2491
2492/// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
Dan Gohman0bba49c2009-07-07 17:06:11 +00002493const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
Dan Gohman622ed672009-05-04 22:02:23 +00002494 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
Owen Anderson73c6b712009-07-13 20:58:05 +00002495 return getConstant(
Owen Andersonbaf3c402009-07-29 18:55:55 +00002496 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
Dan Gohman2d1be872009-04-16 03:18:22 +00002497
2498 const Type *Ty = V->getType();
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002499 Ty = getEffectiveSCEVType(Ty);
Owen Anderson73c6b712009-07-13 20:58:05 +00002500 const SCEV *AllOnes =
Owen Andersona7235ea2009-07-31 20:28:14 +00002501 getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)));
Dan Gohman2d1be872009-04-16 03:18:22 +00002502 return getMinusSCEV(AllOnes, V);
2503}
2504
2505/// getMinusSCEV - Return a SCEV corresponding to LHS - RHS.
2506///
Dan Gohman9311ef62009-06-24 14:49:00 +00002507const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS,
2508 const SCEV *RHS) {
Dan Gohmaneb4152c2010-07-20 16:53:00 +00002509 // Fast path: X - X --> 0.
2510 if (LHS == RHS)
2511 return getConstant(LHS->getType(), 0);
2512
Dan Gohman2d1be872009-04-16 03:18:22 +00002513 // X - Y --> X + -Y
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002514 return getAddExpr(LHS, getNegativeSCEV(RHS));
Dan Gohman2d1be872009-04-16 03:18:22 +00002515}
2516
2517/// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the
2518/// input value to the specified type. If the type must be extended, it is zero
2519/// extended.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002520const SCEV *
2521ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V,
Nick Lewycky5cd28fa2009-04-23 05:15:08 +00002522 const Type *Ty) {
Dan Gohman2d1be872009-04-16 03:18:22 +00002523 const Type *SrcTy = V->getType();
Duncan Sands1df98592010-02-16 11:11:14 +00002524 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
2525 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Dan Gohman2d1be872009-04-16 03:18:22 +00002526 "Cannot truncate or zero extend with non-integer arguments!");
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002527 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
Dan Gohman2d1be872009-04-16 03:18:22 +00002528 return V; // No conversion
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002529 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002530 return getTruncateExpr(V, Ty);
2531 return getZeroExtendExpr(V, Ty);
Dan Gohman2d1be872009-04-16 03:18:22 +00002532}
2533
2534/// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the
2535/// input value to the specified type. If the type must be extended, it is sign
2536/// extended.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002537const SCEV *
2538ScalarEvolution::getTruncateOrSignExtend(const SCEV *V,
Nick Lewycky5cd28fa2009-04-23 05:15:08 +00002539 const Type *Ty) {
Dan Gohman2d1be872009-04-16 03:18:22 +00002540 const Type *SrcTy = V->getType();
Duncan Sands1df98592010-02-16 11:11:14 +00002541 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
2542 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Dan Gohman2d1be872009-04-16 03:18:22 +00002543 "Cannot truncate or zero extend with non-integer arguments!");
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002544 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
Dan Gohman2d1be872009-04-16 03:18:22 +00002545 return V; // No conversion
Dan Gohmanaf79fb52009-04-21 01:07:12 +00002546 if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002547 return getTruncateExpr(V, Ty);
2548 return getSignExtendExpr(V, Ty);
Dan Gohman2d1be872009-04-16 03:18:22 +00002549}
2550
Dan Gohman467c4302009-05-13 03:46:30 +00002551/// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the
2552/// input value to the specified type. If the type must be extended, it is zero
2553/// extended. The conversion must not be narrowing.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002554const SCEV *
2555ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, const Type *Ty) {
Dan Gohman467c4302009-05-13 03:46:30 +00002556 const Type *SrcTy = V->getType();
Duncan Sands1df98592010-02-16 11:11:14 +00002557 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
2558 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Dan Gohman467c4302009-05-13 03:46:30 +00002559 "Cannot noop or zero extend with non-integer arguments!");
2560 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
2561 "getNoopOrZeroExtend cannot truncate!");
2562 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
2563 return V; // No conversion
2564 return getZeroExtendExpr(V, Ty);
2565}
2566
2567/// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the
2568/// input value to the specified type. If the type must be extended, it is sign
2569/// extended. The conversion must not be narrowing.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002570const SCEV *
2571ScalarEvolution::getNoopOrSignExtend(const SCEV *V, const Type *Ty) {
Dan Gohman467c4302009-05-13 03:46:30 +00002572 const Type *SrcTy = V->getType();
Duncan Sands1df98592010-02-16 11:11:14 +00002573 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
2574 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Dan Gohman467c4302009-05-13 03:46:30 +00002575 "Cannot noop or sign extend with non-integer arguments!");
2576 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
2577 "getNoopOrSignExtend cannot truncate!");
2578 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
2579 return V; // No conversion
2580 return getSignExtendExpr(V, Ty);
2581}
2582
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00002583/// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of
2584/// the input value to the specified type. If the type must be extended,
2585/// it is extended with unspecified bits. The conversion must not be
2586/// narrowing.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002587const SCEV *
2588ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, const Type *Ty) {
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00002589 const Type *SrcTy = V->getType();
Duncan Sands1df98592010-02-16 11:11:14 +00002590 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
2591 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Dan Gohman2ce84c8d2009-06-13 15:56:47 +00002592 "Cannot noop or any extend with non-integer arguments!");
2593 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
2594 "getNoopOrAnyExtend cannot truncate!");
2595 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
2596 return V; // No conversion
2597 return getAnyExtendExpr(V, Ty);
2598}
2599
Dan Gohman467c4302009-05-13 03:46:30 +00002600/// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the
2601/// input value to the specified type. The conversion must not be widening.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002602const SCEV *
2603ScalarEvolution::getTruncateOrNoop(const SCEV *V, const Type *Ty) {
Dan Gohman467c4302009-05-13 03:46:30 +00002604 const Type *SrcTy = V->getType();
Duncan Sands1df98592010-02-16 11:11:14 +00002605 assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
2606 (Ty->isIntegerTy() || Ty->isPointerTy()) &&
Dan Gohman467c4302009-05-13 03:46:30 +00002607 "Cannot truncate or noop with non-integer arguments!");
2608 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
2609 "getTruncateOrNoop cannot extend!");
2610 if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
2611 return V; // No conversion
2612 return getTruncateExpr(V, Ty);
2613}
2614
Dan Gohmana334aa72009-06-22 00:31:57 +00002615/// getUMaxFromMismatchedTypes - Promote the operands to the wider of
2616/// the types using zero-extension, and then perform a umax operation
2617/// with them.
Dan Gohman9311ef62009-06-24 14:49:00 +00002618const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
2619 const SCEV *RHS) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00002620 const SCEV *PromotedLHS = LHS;
2621 const SCEV *PromotedRHS = RHS;
Dan Gohmana334aa72009-06-22 00:31:57 +00002622
2623 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
2624 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
2625 else
2626 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
2627
2628 return getUMaxExpr(PromotedLHS, PromotedRHS);
2629}
2630
Dan Gohmanc9759e82009-06-22 15:03:27 +00002631/// getUMinFromMismatchedTypes - Promote the operands to the wider of
2632/// the types using zero-extension, and then perform a umin operation
2633/// with them.
Dan Gohman9311ef62009-06-24 14:49:00 +00002634const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
2635 const SCEV *RHS) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00002636 const SCEV *PromotedLHS = LHS;
2637 const SCEV *PromotedRHS = RHS;
Dan Gohmanc9759e82009-06-22 15:03:27 +00002638
2639 if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
2640 PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
2641 else
2642 PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
2643
2644 return getUMinExpr(PromotedLHS, PromotedRHS);
2645}
2646
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002647/// PushDefUseChildren - Push users of the given Instruction
2648/// onto the given Worklist.
2649static void
2650PushDefUseChildren(Instruction *I,
2651 SmallVectorImpl<Instruction *> &Worklist) {
2652 // Push the def-use children onto the Worklist stack.
2653 for (Value::use_iterator UI = I->use_begin(), UE = I->use_end();
2654 UI != UE; ++UI)
Gabor Greif96f1d8e2010-07-22 13:36:47 +00002655 Worklist.push_back(cast<Instruction>(*UI));
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002656}
2657
2658/// ForgetSymbolicValue - This looks up computed SCEV values for all
2659/// instructions that depend on the given instruction and removes them from
2660/// the Scalars map if they reference SymName. This is used during PHI
2661/// resolution.
Dan Gohman64a845e2009-06-24 04:48:43 +00002662void
Dan Gohman85669632010-02-25 06:57:05 +00002663ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) {
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002664 SmallVector<Instruction *, 16> Worklist;
Dan Gohman85669632010-02-25 06:57:05 +00002665 PushDefUseChildren(PN, Worklist);
Chris Lattner53e677a2004-04-02 20:23:17 +00002666
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002667 SmallPtrSet<Instruction *, 8> Visited;
Dan Gohman85669632010-02-25 06:57:05 +00002668 Visited.insert(PN);
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002669 while (!Worklist.empty()) {
Dan Gohman85669632010-02-25 06:57:05 +00002670 Instruction *I = Worklist.pop_back_val();
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002671 if (!Visited.insert(I)) continue;
Chris Lattner4dc534c2005-02-13 04:37:18 +00002672
Dan Gohman5d984912009-12-18 01:14:11 +00002673 std::map<SCEVCallbackVH, const SCEV *>::iterator It =
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002674 Scalars.find(static_cast<Value *>(I));
2675 if (It != Scalars.end()) {
2676 // Short-circuit the def-use traversal if the symbolic name
2677 // ceases to appear in expressions.
Dan Gohman50922bb2010-02-15 10:28:37 +00002678 if (It->second != SymName && !It->second->hasOperand(SymName))
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002679 continue;
Chris Lattner4dc534c2005-02-13 04:37:18 +00002680
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002681 // SCEVUnknown for a PHI either means that it has an unrecognized
Dan Gohman85669632010-02-25 06:57:05 +00002682 // structure, it's a PHI that's in the progress of being computed
2683 // by createNodeForPHI, or it's a single-value PHI. In the first case,
2684 // additional loop trip count information isn't going to change anything.
2685 // In the second case, createNodeForPHI will perform the necessary
2686 // updates on its own when it gets to that point. In the third, we do
2687 // want to forget the SCEVUnknown.
2688 if (!isa<PHINode>(I) ||
2689 !isa<SCEVUnknown>(It->second) ||
2690 (I != PN && It->second == SymName)) {
Dan Gohman42214892009-08-31 21:15:23 +00002691 ValuesAtScopes.erase(It->second);
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002692 Scalars.erase(It);
Dan Gohman42214892009-08-31 21:15:23 +00002693 }
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002694 }
2695
2696 PushDefUseChildren(I, Worklist);
2697 }
Chris Lattner4dc534c2005-02-13 04:37:18 +00002698}
Chris Lattner53e677a2004-04-02 20:23:17 +00002699
2700/// createNodeForPHI - PHI nodes have two cases. Either the PHI node exists in
2701/// a loop header, making it a potential recurrence, or it doesn't.
2702///
Dan Gohman0bba49c2009-07-07 17:06:11 +00002703const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
Dan Gohman27dead42010-04-12 07:49:36 +00002704 if (const Loop *L = LI->getLoopFor(PN->getParent()))
2705 if (L->getHeader() == PN->getParent()) {
2706 // The loop may have multiple entrances or multiple exits; we can analyze
2707 // this phi as an addrec if it has a unique entry value and a unique
2708 // backedge value.
2709 Value *BEValueV = 0, *StartValueV = 0;
2710 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
2711 Value *V = PN->getIncomingValue(i);
2712 if (L->contains(PN->getIncomingBlock(i))) {
2713 if (!BEValueV) {
2714 BEValueV = V;
2715 } else if (BEValueV != V) {
2716 BEValueV = 0;
2717 break;
2718 }
2719 } else if (!StartValueV) {
2720 StartValueV = V;
2721 } else if (StartValueV != V) {
2722 StartValueV = 0;
2723 break;
2724 }
2725 }
2726 if (BEValueV && StartValueV) {
Chris Lattner53e677a2004-04-02 20:23:17 +00002727 // While we are analyzing this PHI node, handle its value symbolically.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002728 const SCEV *SymbolicName = getUnknown(PN);
Chris Lattner53e677a2004-04-02 20:23:17 +00002729 assert(Scalars.find(PN) == Scalars.end() &&
2730 "PHI node already processed?");
Dan Gohman35738ac2009-05-04 22:30:44 +00002731 Scalars.insert(std::make_pair(SCEVCallbackVH(PN, this), SymbolicName));
Chris Lattner53e677a2004-04-02 20:23:17 +00002732
2733 // Using this symbolic name for the PHI, analyze the value coming around
2734 // the back-edge.
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002735 const SCEV *BEValue = getSCEV(BEValueV);
Chris Lattner53e677a2004-04-02 20:23:17 +00002736
2737 // NOTE: If BEValue is loop invariant, we know that the PHI node just
2738 // has a special value for the first iteration of the loop.
2739
2740 // If the value coming around the backedge is an add with the symbolic
2741 // value we just inserted, then we found a simple induction variable!
Dan Gohman622ed672009-05-04 22:02:23 +00002742 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00002743 // If there is a single occurrence of the symbolic value, replace it
2744 // with a recurrence.
2745 unsigned FoundIndex = Add->getNumOperands();
2746 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
2747 if (Add->getOperand(i) == SymbolicName)
2748 if (FoundIndex == e) {
2749 FoundIndex = i;
2750 break;
2751 }
2752
2753 if (FoundIndex != Add->getNumOperands()) {
2754 // Create an add with everything but the specified operand.
Dan Gohman0bba49c2009-07-07 17:06:11 +00002755 SmallVector<const SCEV *, 8> Ops;
Chris Lattner53e677a2004-04-02 20:23:17 +00002756 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
2757 if (i != FoundIndex)
2758 Ops.push_back(Add->getOperand(i));
Dan Gohman0bba49c2009-07-07 17:06:11 +00002759 const SCEV *Accum = getAddExpr(Ops);
Chris Lattner53e677a2004-04-02 20:23:17 +00002760
2761 // This is not a valid addrec if the step amount is varying each
2762 // loop iteration, but is not itself an addrec in this loop.
2763 if (Accum->isLoopInvariant(L) ||
2764 (isa<SCEVAddRecExpr>(Accum) &&
2765 cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
Dan Gohmana10756e2010-01-21 02:09:26 +00002766 bool HasNUW = false;
2767 bool HasNSW = false;
2768
2769 // If the increment doesn't overflow, then neither the addrec nor
2770 // the post-increment will overflow.
2771 if (const AddOperator *OBO = dyn_cast<AddOperator>(BEValueV)) {
2772 if (OBO->hasNoUnsignedWrap())
2773 HasNUW = true;
2774 if (OBO->hasNoSignedWrap())
2775 HasNSW = true;
2776 }
2777
Dan Gohman27dead42010-04-12 07:49:36 +00002778 const SCEV *StartVal = getSCEV(StartValueV);
Dan Gohmana10756e2010-01-21 02:09:26 +00002779 const SCEV *PHISCEV =
2780 getAddRecExpr(StartVal, Accum, L, HasNUW, HasNSW);
Dan Gohmaneb490a72009-07-25 01:22:26 +00002781
Dan Gohmana10756e2010-01-21 02:09:26 +00002782 // Since the no-wrap flags are on the increment, they apply to the
2783 // post-incremented value as well.
2784 if (Accum->isLoopInvariant(L))
2785 (void)getAddRecExpr(getAddExpr(StartVal, Accum),
2786 Accum, L, HasNUW, HasNSW);
Chris Lattner53e677a2004-04-02 20:23:17 +00002787
2788 // Okay, for the entire analysis of this edge we assumed the PHI
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002789 // to be symbolic. We now need to go back and purge all of the
2790 // entries for the scalars that use the symbolic expression.
2791 ForgetSymbolicName(PN, SymbolicName);
2792 Scalars[SCEVCallbackVH(PN, this)] = PHISCEV;
Chris Lattner53e677a2004-04-02 20:23:17 +00002793 return PHISCEV;
2794 }
2795 }
Dan Gohman622ed672009-05-04 22:02:23 +00002796 } else if (const SCEVAddRecExpr *AddRec =
2797 dyn_cast<SCEVAddRecExpr>(BEValue)) {
Chris Lattner97156e72006-04-26 18:34:07 +00002798 // Otherwise, this could be a loop like this:
2799 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
2800 // In this case, j = {1,+,1} and BEValue is j.
2801 // Because the other in-value of i (0) fits the evolution of BEValue
2802 // i really is an addrec evolution.
2803 if (AddRec->getLoop() == L && AddRec->isAffine()) {
Dan Gohman27dead42010-04-12 07:49:36 +00002804 const SCEV *StartVal = getSCEV(StartValueV);
Chris Lattner97156e72006-04-26 18:34:07 +00002805
2806 // If StartVal = j.start - j.stride, we can use StartVal as the
2807 // initial step of the addrec evolution.
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002808 if (StartVal == getMinusSCEV(AddRec->getOperand(0),
Dan Gohman5ee60f72010-04-11 23:44:58 +00002809 AddRec->getOperand(1))) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00002810 const SCEV *PHISCEV =
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002811 getAddRecExpr(StartVal, AddRec->getOperand(1), L);
Chris Lattner97156e72006-04-26 18:34:07 +00002812
2813 // Okay, for the entire analysis of this edge we assumed the PHI
Dan Gohmanfef8bb22009-07-25 01:13:03 +00002814 // to be symbolic. We now need to go back and purge all of the
2815 // entries for the scalars that use the symbolic expression.
2816 ForgetSymbolicName(PN, SymbolicName);
2817 Scalars[SCEVCallbackVH(PN, this)] = PHISCEV;
Chris Lattner97156e72006-04-26 18:34:07 +00002818 return PHISCEV;
2819 }
2820 }
Chris Lattner53e677a2004-04-02 20:23:17 +00002821 }
Chris Lattner53e677a2004-04-02 20:23:17 +00002822 }
Dan Gohman27dead42010-04-12 07:49:36 +00002823 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00002824
Dan Gohman85669632010-02-25 06:57:05 +00002825 // If the PHI has a single incoming value, follow that value, unless the
2826 // PHI's incoming blocks are in a different loop, in which case doing so
2827 // risks breaking LCSSA form. Instcombine would normally zap these, but
2828 // it doesn't have DominatorTree information, so it may miss cases.
2829 if (Value *V = PN->hasConstantValue(DT)) {
2830 bool AllSameLoop = true;
2831 Loop *PNLoop = LI->getLoopFor(PN->getParent());
2832 for (size_t i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
2833 if (LI->getLoopFor(PN->getIncomingBlock(i)) != PNLoop) {
2834 AllSameLoop = false;
2835 break;
2836 }
2837 if (AllSameLoop)
2838 return getSCEV(V);
2839 }
Dan Gohmana653fc52009-07-14 14:06:25 +00002840
Chris Lattner53e677a2004-04-02 20:23:17 +00002841 // If it's not a loop phi, we can't handle it yet.
Dan Gohmanf8a8be82009-04-21 23:15:49 +00002842 return getUnknown(PN);
Chris Lattner53e677a2004-04-02 20:23:17 +00002843}
2844
Dan Gohman26466c02009-05-08 20:26:55 +00002845/// createNodeForGEP - Expand GEP instructions into add and multiply
2846/// operations. This allows them to be analyzed by regular SCEV code.
2847///
Dan Gohmand281ed22009-12-18 02:09:29 +00002848const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
Dan Gohman26466c02009-05-08 20:26:55 +00002849
Dan Gohmanb9f96512010-06-30 07:16:37 +00002850 // Don't blindly transfer the inbounds flag from the GEP instruction to the
2851 // Add expression, because the Instruction may be guarded by control flow
2852 // and the no-overflow bits may not be valid for the expression in any
Dan Gohman70eff632010-06-30 17:27:11 +00002853 // context.
Dan Gohman7a642572010-06-29 01:41:41 +00002854
Dan Gohmanc40f17b2009-08-18 16:46:41 +00002855 const Type *IntPtrTy = getEffectiveSCEVType(GEP->getType());
Dan Gohmane810b0d2009-05-08 20:36:47 +00002856 Value *Base = GEP->getOperand(0);
Dan Gohmanc63a6272009-05-09 00:14:52 +00002857 // Don't attempt to analyze GEPs over unsized objects.
2858 if (!cast<PointerType>(Base->getType())->getElementType()->isSized())
2859 return getUnknown(GEP);
Dan Gohmandeff6212010-05-03 22:09:21 +00002860 const SCEV *TotalOffset = getConstant(IntPtrTy, 0);
Dan Gohmane810b0d2009-05-08 20:36:47 +00002861 gep_type_iterator GTI = gep_type_begin(GEP);
Oscar Fuentesee56c422010-08-02 06:00:15 +00002862 for (GetElementPtrInst::op_iterator I = llvm::next(GEP->op_begin()),
Dan Gohmane810b0d2009-05-08 20:36:47 +00002863 E = GEP->op_end();
Dan Gohman26466c02009-05-08 20:26:55 +00002864 I != E; ++I) {
2865 Value *Index = *I;
2866 // Compute the (potentially symbolic) offset in bytes for this index.
2867 if (const StructType *STy = dyn_cast<StructType>(*GTI++)) {
2868 // For a struct, add the member offset.
Dan Gohman26466c02009-05-08 20:26:55 +00002869 unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue();
Dan Gohmanb9f96512010-06-30 07:16:37 +00002870 const SCEV *FieldOffset = getOffsetOfExpr(STy, FieldNo);
2871
Dan Gohmanb9f96512010-06-30 07:16:37 +00002872 // Add the field offset to the running total offset.
Dan Gohman70eff632010-06-30 17:27:11 +00002873 TotalOffset = getAddExpr(TotalOffset, FieldOffset);
Dan Gohman26466c02009-05-08 20:26:55 +00002874 } else {
2875 // For an array, add the element offset, explicitly scaled.
Dan Gohmanb9f96512010-06-30 07:16:37 +00002876 const SCEV *ElementSize = getSizeOfExpr(*GTI);
2877 const SCEV *IndexS = getSCEV(Index);
Dan Gohman3f46a3a2010-03-01 17:49:51 +00002878 // Getelementptr indices are signed.
Dan Gohmanb9f96512010-06-30 07:16:37 +00002879 IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy);
2880
Dan Gohmanb9f96512010-06-30 07:16:37 +00002881 // Multiply the index by the element size to compute the element offset.
Dan Gohman70eff632010-06-30 17:27:11 +00002882 const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize);
Dan Gohmanb9f96512010-06-30 07:16:37 +00002883
2884 // Add the element offset to the running total offset.
Dan Gohman70eff632010-06-30 17:27:11 +00002885 TotalOffset = getAddExpr(TotalOffset, LocalOffset);
Dan Gohman26466c02009-05-08 20:26:55 +00002886 }
2887 }
Dan Gohmanb9f96512010-06-30 07:16:37 +00002888
2889 // Get the SCEV for the GEP base.
2890 const SCEV *BaseS = getSCEV(Base);
2891
Dan Gohmanb9f96512010-06-30 07:16:37 +00002892 // Add the total offset from all the GEP indices to the base.
Dan Gohman70eff632010-06-30 17:27:11 +00002893 return getAddExpr(BaseS, TotalOffset);
Dan Gohman26466c02009-05-08 20:26:55 +00002894}
2895
Nick Lewycky83bb0052007-11-22 07:59:40 +00002896/// GetMinTrailingZeros - Determine the minimum number of zero bits that S is
2897/// guaranteed to end in (at every loop iteration). It is, at the same time,
2898/// the minimum number of times S is divisible by 2. For example, given {4,+,8}
2899/// it returns 2. If S is guaranteed to be 0, it returns the bitwidth of S.
Dan Gohman2c364ad2009-06-19 23:29:04 +00002900uint32_t
Dan Gohman0bba49c2009-07-07 17:06:11 +00002901ScalarEvolution::GetMinTrailingZeros(const SCEV *S) {
Dan Gohman622ed672009-05-04 22:02:23 +00002902 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
Chris Lattner8314a0c2007-11-23 22:36:49 +00002903 return C->getValue()->getValue().countTrailingZeros();
Chris Lattnera17f0392006-12-12 02:26:09 +00002904
Dan Gohman622ed672009-05-04 22:02:23 +00002905 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S))
Dan Gohman2c364ad2009-06-19 23:29:04 +00002906 return std::min(GetMinTrailingZeros(T->getOperand()),
2907 (uint32_t)getTypeSizeInBits(T->getType()));
Nick Lewycky83bb0052007-11-22 07:59:40 +00002908
Dan Gohman622ed672009-05-04 22:02:23 +00002909 if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) {
Dan Gohman2c364ad2009-06-19 23:29:04 +00002910 uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
2911 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
2912 getTypeSizeInBits(E->getType()) : OpRes;
Nick Lewycky83bb0052007-11-22 07:59:40 +00002913 }
2914
Dan Gohman622ed672009-05-04 22:02:23 +00002915 if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) {
Dan Gohman2c364ad2009-06-19 23:29:04 +00002916 uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
2917 return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
2918 getTypeSizeInBits(E->getType()) : OpRes;
Nick Lewycky83bb0052007-11-22 07:59:40 +00002919 }
2920
Dan Gohman622ed672009-05-04 22:02:23 +00002921 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) {
Nick Lewycky83bb0052007-11-22 07:59:40 +00002922 // The result is the min of all operands results.
Dan Gohman2c364ad2009-06-19 23:29:04 +00002923 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
Nick Lewycky83bb0052007-11-22 07:59:40 +00002924 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
Dan Gohman2c364ad2009-06-19 23:29:04 +00002925 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
Nick Lewycky83bb0052007-11-22 07:59:40 +00002926 return MinOpRes;
Chris Lattnera17f0392006-12-12 02:26:09 +00002927 }
2928
Dan Gohman622ed672009-05-04 22:02:23 +00002929 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
Nick Lewycky83bb0052007-11-22 07:59:40 +00002930 // The result is the sum of all operands results.
Dan Gohman2c364ad2009-06-19 23:29:04 +00002931 uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0));
2932 uint32_t BitWidth = getTypeSizeInBits(M->getType());
Nick Lewycky83bb0052007-11-22 07:59:40 +00002933 for (unsigned i = 1, e = M->getNumOperands();
2934 SumOpRes != BitWidth && i != e; ++i)
Dan Gohman2c364ad2009-06-19 23:29:04 +00002935 SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)),
Nick Lewycky83bb0052007-11-22 07:59:40 +00002936 BitWidth);
2937 return SumOpRes;
Chris Lattnera17f0392006-12-12 02:26:09 +00002938 }
Nick Lewycky83bb0052007-11-22 07:59:40 +00002939
Dan Gohman622ed672009-05-04 22:02:23 +00002940 if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) {
Nick Lewycky83bb0052007-11-22 07:59:40 +00002941 // The result is the min of all operands results.
Dan Gohman2c364ad2009-06-19 23:29:04 +00002942 uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
Nick Lewycky83bb0052007-11-22 07:59:40 +00002943 for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
Dan Gohman2c364ad2009-06-19 23:29:04 +00002944 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
Nick Lewycky83bb0052007-11-22 07:59:40 +00002945 return MinOpRes;
Chris Lattnera17f0392006-12-12 02:26:09 +00002946 }
Nick Lewycky83bb0052007-11-22 07:59:40 +00002947
Dan Gohman622ed672009-05-04 22:02:23 +00002948 if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) {
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002949 // The result is the min of all operands results.
Dan Gohman2c364ad2009-06-19 23:29:04 +00002950 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002951 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
Dan Gohman2c364ad2009-06-19 23:29:04 +00002952 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
Nick Lewyckyc54c5612007-11-25 22:41:31 +00002953 return MinOpRes;
2954 }
2955
Dan Gohman622ed672009-05-04 22:02:23 +00002956 if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) {
Nick Lewycky3e630762008-02-20 06:48:22 +00002957 // The result is the min of all operands results.
Dan Gohman2c364ad2009-06-19 23:29:04 +00002958 uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
Nick Lewycky3e630762008-02-20 06:48:22 +00002959 for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
Dan Gohman2c364ad2009-06-19 23:29:04 +00002960 MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
Nick Lewycky3e630762008-02-20 06:48:22 +00002961 return MinOpRes;
2962 }
2963
Dan Gohman2c364ad2009-06-19 23:29:04 +00002964 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
2965 // For a SCEVUnknown, ask ValueTracking.
2966 unsigned BitWidth = getTypeSizeInBits(U->getType());
2967 APInt Mask = APInt::getAllOnesValue(BitWidth);
2968 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
2969 ComputeMaskedBits(U->getValue(), Mask, Zeros, Ones);
2970 return Zeros.countTrailingOnes();
2971 }
2972
2973 // SCEVUDivExpr
Nick Lewycky83bb0052007-11-22 07:59:40 +00002974 return 0;
Chris Lattnera17f0392006-12-12 02:26:09 +00002975}
Chris Lattner53e677a2004-04-02 20:23:17 +00002976
Dan Gohman85b05a22009-07-13 21:35:55 +00002977/// getUnsignedRange - Determine the unsigned range for a particular SCEV.
2978///
2979ConstantRange
2980ScalarEvolution::getUnsignedRange(const SCEV *S) {
Dan Gohman2c364ad2009-06-19 23:29:04 +00002981
2982 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
Dan Gohman85b05a22009-07-13 21:35:55 +00002983 return ConstantRange(C->getValue()->getValue());
Dan Gohman2c364ad2009-06-19 23:29:04 +00002984
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00002985 unsigned BitWidth = getTypeSizeInBits(S->getType());
2986 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
2987
2988 // If the value has known zeros, the maximum unsigned value will have those
2989 // known zeros as well.
2990 uint32_t TZ = GetMinTrailingZeros(S);
2991 if (TZ != 0)
2992 ConservativeResult =
2993 ConstantRange(APInt::getMinValue(BitWidth),
2994 APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1);
2995
Dan Gohman85b05a22009-07-13 21:35:55 +00002996 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
2997 ConstantRange X = getUnsignedRange(Add->getOperand(0));
2998 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
2999 X = X.add(getUnsignedRange(Add->getOperand(i)));
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003000 return ConservativeResult.intersectWith(X);
Dan Gohman85b05a22009-07-13 21:35:55 +00003001 }
3002
3003 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
3004 ConstantRange X = getUnsignedRange(Mul->getOperand(0));
3005 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
3006 X = X.multiply(getUnsignedRange(Mul->getOperand(i)));
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003007 return ConservativeResult.intersectWith(X);
Dan Gohman85b05a22009-07-13 21:35:55 +00003008 }
3009
3010 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
3011 ConstantRange X = getUnsignedRange(SMax->getOperand(0));
3012 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i)
3013 X = X.smax(getUnsignedRange(SMax->getOperand(i)));
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003014 return ConservativeResult.intersectWith(X);
Dan Gohman85b05a22009-07-13 21:35:55 +00003015 }
3016
3017 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
3018 ConstantRange X = getUnsignedRange(UMax->getOperand(0));
3019 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i)
3020 X = X.umax(getUnsignedRange(UMax->getOperand(i)));
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003021 return ConservativeResult.intersectWith(X);
Dan Gohman85b05a22009-07-13 21:35:55 +00003022 }
3023
3024 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
3025 ConstantRange X = getUnsignedRange(UDiv->getLHS());
3026 ConstantRange Y = getUnsignedRange(UDiv->getRHS());
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003027 return ConservativeResult.intersectWith(X.udiv(Y));
Dan Gohman85b05a22009-07-13 21:35:55 +00003028 }
3029
3030 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
3031 ConstantRange X = getUnsignedRange(ZExt->getOperand());
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003032 return ConservativeResult.intersectWith(X.zeroExtend(BitWidth));
Dan Gohman85b05a22009-07-13 21:35:55 +00003033 }
3034
3035 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
3036 ConstantRange X = getUnsignedRange(SExt->getOperand());
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003037 return ConservativeResult.intersectWith(X.signExtend(BitWidth));
Dan Gohman85b05a22009-07-13 21:35:55 +00003038 }
3039
3040 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
3041 ConstantRange X = getUnsignedRange(Trunc->getOperand());
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003042 return ConservativeResult.intersectWith(X.truncate(BitWidth));
Dan Gohman85b05a22009-07-13 21:35:55 +00003043 }
3044
Dan Gohman85b05a22009-07-13 21:35:55 +00003045 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
Dan Gohmana10756e2010-01-21 02:09:26 +00003046 // If there's no unsigned wrap, the value will never be less than its
3047 // initial value.
3048 if (AddRec->hasNoUnsignedWrap())
3049 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart()))
Dan Gohmanbca091d2010-04-12 23:08:18 +00003050 if (!C->getValue()->isZero())
Dan Gohmanbc7129f2010-04-11 22:12:18 +00003051 ConservativeResult =
Dan Gohman8a18d6b2010-06-30 06:58:35 +00003052 ConservativeResult.intersectWith(
3053 ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0)));
Dan Gohman85b05a22009-07-13 21:35:55 +00003054
3055 // TODO: non-affine addrec
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003056 if (AddRec->isAffine()) {
Dan Gohman85b05a22009-07-13 21:35:55 +00003057 const Type *Ty = AddRec->getType();
3058 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003059 if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
3060 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
Dan Gohman85b05a22009-07-13 21:35:55 +00003061 MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty);
3062
3063 const SCEV *Start = AddRec->getStart();
Dan Gohman646e0472010-04-12 07:39:33 +00003064 const SCEV *Step = AddRec->getStepRecurrence(*this);
Dan Gohman85b05a22009-07-13 21:35:55 +00003065
3066 ConstantRange StartRange = getUnsignedRange(Start);
Dan Gohman646e0472010-04-12 07:39:33 +00003067 ConstantRange StepRange = getSignedRange(Step);
3068 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount);
3069 ConstantRange EndRange =
3070 StartRange.add(MaxBECountRange.multiply(StepRange));
3071
3072 // Check for overflow. This must be done with ConstantRange arithmetic
3073 // because we could be called from within the ScalarEvolution overflow
3074 // checking code.
3075 ConstantRange ExtStartRange = StartRange.zextOrTrunc(BitWidth*2+1);
3076 ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1);
3077 ConstantRange ExtMaxBECountRange =
3078 MaxBECountRange.zextOrTrunc(BitWidth*2+1);
3079 ConstantRange ExtEndRange = EndRange.zextOrTrunc(BitWidth*2+1);
3080 if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) !=
3081 ExtEndRange)
3082 return ConservativeResult;
3083
Dan Gohman85b05a22009-07-13 21:35:55 +00003084 APInt Min = APIntOps::umin(StartRange.getUnsignedMin(),
3085 EndRange.getUnsignedMin());
3086 APInt Max = APIntOps::umax(StartRange.getUnsignedMax(),
3087 EndRange.getUnsignedMax());
3088 if (Min.isMinValue() && Max.isMaxValue())
Dan Gohmana10756e2010-01-21 02:09:26 +00003089 return ConservativeResult;
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003090 return ConservativeResult.intersectWith(ConstantRange(Min, Max+1));
Dan Gohman85b05a22009-07-13 21:35:55 +00003091 }
3092 }
Dan Gohmana10756e2010-01-21 02:09:26 +00003093
3094 return ConservativeResult;
Dan Gohman2c364ad2009-06-19 23:29:04 +00003095 }
3096
3097 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
3098 // For a SCEVUnknown, ask ValueTracking.
Dan Gohman2c364ad2009-06-19 23:29:04 +00003099 APInt Mask = APInt::getAllOnesValue(BitWidth);
3100 APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
3101 ComputeMaskedBits(U->getValue(), Mask, Zeros, Ones, TD);
Dan Gohman746f3b12009-07-20 22:34:18 +00003102 if (Ones == ~Zeros + 1)
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003103 return ConservativeResult;
3104 return ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1));
Dan Gohman2c364ad2009-06-19 23:29:04 +00003105 }
3106
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003107 return ConservativeResult;
Dan Gohman2c364ad2009-06-19 23:29:04 +00003108}
3109
Dan Gohman85b05a22009-07-13 21:35:55 +00003110/// getSignedRange - Determine the signed range for a particular SCEV.
3111///
3112ConstantRange
3113ScalarEvolution::getSignedRange(const SCEV *S) {
Dan Gohman2c364ad2009-06-19 23:29:04 +00003114
Dan Gohman85b05a22009-07-13 21:35:55 +00003115 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
3116 return ConstantRange(C->getValue()->getValue());
3117
Dan Gohman52fddd32010-01-26 04:40:18 +00003118 unsigned BitWidth = getTypeSizeInBits(S->getType());
3119 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
3120
3121 // If the value has known zeros, the maximum signed value will have those
3122 // known zeros as well.
3123 uint32_t TZ = GetMinTrailingZeros(S);
3124 if (TZ != 0)
3125 ConservativeResult =
3126 ConstantRange(APInt::getSignedMinValue(BitWidth),
3127 APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
3128
Dan Gohman85b05a22009-07-13 21:35:55 +00003129 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
3130 ConstantRange X = getSignedRange(Add->getOperand(0));
3131 for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
3132 X = X.add(getSignedRange(Add->getOperand(i)));
Dan Gohman52fddd32010-01-26 04:40:18 +00003133 return ConservativeResult.intersectWith(X);
Dan Gohman2c364ad2009-06-19 23:29:04 +00003134 }
3135
Dan Gohman85b05a22009-07-13 21:35:55 +00003136 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
3137 ConstantRange X = getSignedRange(Mul->getOperand(0));
3138 for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
3139 X = X.multiply(getSignedRange(Mul->getOperand(i)));
Dan Gohman52fddd32010-01-26 04:40:18 +00003140 return ConservativeResult.intersectWith(X);
Dan Gohman2c364ad2009-06-19 23:29:04 +00003141 }
3142
Dan Gohman85b05a22009-07-13 21:35:55 +00003143 if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
3144 ConstantRange X = getSignedRange(SMax->getOperand(0));
3145 for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i)
3146 X = X.smax(getSignedRange(SMax->getOperand(i)));
Dan Gohman52fddd32010-01-26 04:40:18 +00003147 return ConservativeResult.intersectWith(X);
Dan Gohman85b05a22009-07-13 21:35:55 +00003148 }
Dan Gohman62849c02009-06-24 01:05:09 +00003149
Dan Gohman85b05a22009-07-13 21:35:55 +00003150 if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
3151 ConstantRange X = getSignedRange(UMax->getOperand(0));
3152 for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i)
3153 X = X.umax(getSignedRange(UMax->getOperand(i)));
Dan Gohman52fddd32010-01-26 04:40:18 +00003154 return ConservativeResult.intersectWith(X);
Dan Gohman85b05a22009-07-13 21:35:55 +00003155 }
Dan Gohman62849c02009-06-24 01:05:09 +00003156
Dan Gohman85b05a22009-07-13 21:35:55 +00003157 if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
3158 ConstantRange X = getSignedRange(UDiv->getLHS());
3159 ConstantRange Y = getSignedRange(UDiv->getRHS());
Dan Gohman52fddd32010-01-26 04:40:18 +00003160 return ConservativeResult.intersectWith(X.udiv(Y));
Dan Gohman85b05a22009-07-13 21:35:55 +00003161 }
Dan Gohman62849c02009-06-24 01:05:09 +00003162
Dan Gohman85b05a22009-07-13 21:35:55 +00003163 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
3164 ConstantRange X = getSignedRange(ZExt->getOperand());
Dan Gohman52fddd32010-01-26 04:40:18 +00003165 return ConservativeResult.intersectWith(X.zeroExtend(BitWidth));
Dan Gohman85b05a22009-07-13 21:35:55 +00003166 }
3167
3168 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
3169 ConstantRange X = getSignedRange(SExt->getOperand());
Dan Gohman52fddd32010-01-26 04:40:18 +00003170 return ConservativeResult.intersectWith(X.signExtend(BitWidth));
Dan Gohman85b05a22009-07-13 21:35:55 +00003171 }
3172
3173 if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
3174 ConstantRange X = getSignedRange(Trunc->getOperand());
Dan Gohman52fddd32010-01-26 04:40:18 +00003175 return ConservativeResult.intersectWith(X.truncate(BitWidth));
Dan Gohman85b05a22009-07-13 21:35:55 +00003176 }
3177
Dan Gohman85b05a22009-07-13 21:35:55 +00003178 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
Dan Gohmana10756e2010-01-21 02:09:26 +00003179 // If there's no signed wrap, and all the operands have the same sign or
3180 // zero, the value won't ever change sign.
3181 if (AddRec->hasNoSignedWrap()) {
3182 bool AllNonNeg = true;
3183 bool AllNonPos = true;
3184 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3185 if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false;
3186 if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false;
3187 }
Dan Gohmana10756e2010-01-21 02:09:26 +00003188 if (AllNonNeg)
Dan Gohman52fddd32010-01-26 04:40:18 +00003189 ConservativeResult = ConservativeResult.intersectWith(
3190 ConstantRange(APInt(BitWidth, 0),
3191 APInt::getSignedMinValue(BitWidth)));
Dan Gohmana10756e2010-01-21 02:09:26 +00003192 else if (AllNonPos)
Dan Gohman52fddd32010-01-26 04:40:18 +00003193 ConservativeResult = ConservativeResult.intersectWith(
3194 ConstantRange(APInt::getSignedMinValue(BitWidth),
3195 APInt(BitWidth, 1)));
Dan Gohmana10756e2010-01-21 02:09:26 +00003196 }
Dan Gohman85b05a22009-07-13 21:35:55 +00003197
3198 // TODO: non-affine addrec
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003199 if (AddRec->isAffine()) {
Dan Gohman85b05a22009-07-13 21:35:55 +00003200 const Type *Ty = AddRec->getType();
3201 const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
Dan Gohmanc9c36cb2010-01-26 19:19:05 +00003202 if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
3203 getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
Dan Gohman85b05a22009-07-13 21:35:55 +00003204 MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty);
3205
3206 const SCEV *Start = AddRec->getStart();
Dan Gohman646e0472010-04-12 07:39:33 +00003207 const SCEV *Step = AddRec->getStepRecurrence(*this);
Dan Gohman85b05a22009-07-13 21:35:55 +00003208
3209 ConstantRange StartRange = getSignedRange(Start);
Dan Gohman646e0472010-04-12 07:39:33 +00003210 ConstantRange StepRange = getSignedRange(Step);
3211 ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount);
3212 ConstantRange EndRange =
3213 StartRange.add(MaxBECountRange.multiply(StepRange));
3214
3215 // Check for overflow. This must be done with ConstantRange arithmetic
3216 // because we could be called from within the ScalarEvolution overflow
3217 // checking code.
3218 ConstantRange ExtStartRange = StartRange.sextOrTrunc(BitWidth*2+1);
3219 ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1);
3220 ConstantRange ExtMaxBECountRange =
3221 MaxBECountRange.zextOrTrunc(BitWidth*2+1);
3222 ConstantRange ExtEndRange = EndRange.sextOrTrunc(BitWidth*2+1);
3223 if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) !=
3224 ExtEndRange)
3225 return ConservativeResult;
3226
Dan Gohman85b05a22009-07-13 21:35:55 +00003227 APInt Min = APIntOps::smin(StartRange.getSignedMin(),
3228 EndRange.getSignedMin());
3229 APInt Max = APIntOps::smax(StartRange.getSignedMax(),
3230 EndRange.getSignedMax());
3231 if (Min.isMinSignedValue() && Max.isMaxSignedValue())
Dan Gohmana10756e2010-01-21 02:09:26 +00003232 return ConservativeResult;
Dan Gohman52fddd32010-01-26 04:40:18 +00003233 return ConservativeResult.intersectWith(ConstantRange(Min, Max+1));
Dan Gohman62849c02009-06-24 01:05:09 +00003234 }
Dan Gohman62849c02009-06-24 01:05:09 +00003235 }
Dan Gohmana10756e2010-01-21 02:09:26 +00003236
3237 return ConservativeResult;
Dan Gohman62849c02009-06-24 01:05:09 +00003238 }
3239
Dan Gohman2c364ad2009-06-19 23:29:04 +00003240 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
3241 // For a SCEVUnknown, ask ValueTracking.
Duncan Sandsb0bc6c32010-02-15 16:12:20 +00003242 if (!U->getValue()->getType()->isIntegerTy() && !TD)
Dan Gohman52fddd32010-01-26 04:40:18 +00003243 return ConservativeResult;
Dan Gohman85b05a22009-07-13 21:35:55 +00003244 unsigned NS = ComputeNumSignBits(U->getValue(), TD);
3245 if (NS == 1)
Dan Gohman52fddd32010-01-26 04:40:18 +00003246 return ConservativeResult;
3247 return ConservativeResult.intersectWith(
Dan Gohman85b05a22009-07-13 21:35:55 +00003248 ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
Dan Gohman52fddd32010-01-26 04:40:18 +00003249 APInt::getSignedMaxValue(BitWidth).ashr(NS - 1)+1));
Dan Gohman2c364ad2009-06-19 23:29:04 +00003250 }
3251
Dan Gohman52fddd32010-01-26 04:40:18 +00003252 return ConservativeResult;
Dan Gohman2c364ad2009-06-19 23:29:04 +00003253}
3254
Chris Lattner53e677a2004-04-02 20:23:17 +00003255/// createSCEV - We know that there is no SCEV for the specified value.
3256/// Analyze the expression.
3257///
Dan Gohman0bba49c2009-07-07 17:06:11 +00003258const SCEV *ScalarEvolution::createSCEV(Value *V) {
Dan Gohmanaf79fb52009-04-21 01:07:12 +00003259 if (!isSCEVable(V->getType()))
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003260 return getUnknown(V);
Dan Gohman2d1be872009-04-16 03:18:22 +00003261
Dan Gohman6c459a22008-06-22 19:56:46 +00003262 unsigned Opcode = Instruction::UserOp1;
Dan Gohman4ecbca52010-03-09 23:46:50 +00003263 if (Instruction *I = dyn_cast<Instruction>(V)) {
Dan Gohman6c459a22008-06-22 19:56:46 +00003264 Opcode = I->getOpcode();
Dan Gohman4ecbca52010-03-09 23:46:50 +00003265
3266 // Don't attempt to analyze instructions in blocks that aren't
3267 // reachable. Such instructions don't matter, and they aren't required
3268 // to obey basic rules for definitions dominating uses which this
3269 // analysis depends on.
3270 if (!DT->isReachableFromEntry(I->getParent()))
3271 return getUnknown(V);
3272 } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
Dan Gohman6c459a22008-06-22 19:56:46 +00003273 Opcode = CE->getOpcode();
Dan Gohman6bbcba12009-06-24 00:54:57 +00003274 else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
3275 return getConstant(CI);
3276 else if (isa<ConstantPointerNull>(V))
Dan Gohmandeff6212010-05-03 22:09:21 +00003277 return getConstant(V->getType(), 0);
Dan Gohman26812322009-08-25 17:49:57 +00003278 else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
3279 return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee());
Dan Gohman6c459a22008-06-22 19:56:46 +00003280 else
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003281 return getUnknown(V);
Chris Lattner2811f2a2007-04-02 05:41:38 +00003282
Dan Gohmanca178902009-07-17 20:47:02 +00003283 Operator *U = cast<Operator>(V);
Dan Gohman6c459a22008-06-22 19:56:46 +00003284 switch (Opcode) {
Dan Gohmand3f171d2010-08-16 16:03:49 +00003285 case Instruction::Add: {
3286 // The simple thing to do would be to just call getSCEV on both operands
3287 // and call getAddExpr with the result. However if we're looking at a
3288 // bunch of things all added together, this can be quite inefficient,
3289 // because it leads to N-1 getAddExpr calls for N ultimate operands.
3290 // Instead, gather up all the operands and make a single getAddExpr call.
3291 // LLVM IR canonical form means we need only traverse the left operands.
3292 SmallVector<const SCEV *, 4> AddOps;
3293 AddOps.push_back(getSCEV(U->getOperand(1)));
3294 for (Value *Op = U->getOperand(0);
3295 Op->getValueID() == Instruction::Add + Value::InstructionVal;
3296 Op = U->getOperand(0)) {
3297 U = cast<Operator>(Op);
3298 AddOps.push_back(getSCEV(U->getOperand(1)));
3299 }
3300 AddOps.push_back(getSCEV(U->getOperand(0)));
3301 return getAddExpr(AddOps);
3302 }
3303 case Instruction::Mul: {
3304 // See the Add code above.
3305 SmallVector<const SCEV *, 4> MulOps;
3306 MulOps.push_back(getSCEV(U->getOperand(1)));
3307 for (Value *Op = U->getOperand(0);
3308 Op->getValueID() == Instruction::Mul + Value::InstructionVal;
3309 Op = U->getOperand(0)) {
3310 U = cast<Operator>(Op);
3311 MulOps.push_back(getSCEV(U->getOperand(1)));
3312 }
3313 MulOps.push_back(getSCEV(U->getOperand(0)));
3314 return getMulExpr(MulOps);
3315 }
Dan Gohman6c459a22008-06-22 19:56:46 +00003316 case Instruction::UDiv:
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003317 return getUDivExpr(getSCEV(U->getOperand(0)),
3318 getSCEV(U->getOperand(1)));
Dan Gohman6c459a22008-06-22 19:56:46 +00003319 case Instruction::Sub:
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003320 return getMinusSCEV(getSCEV(U->getOperand(0)),
3321 getSCEV(U->getOperand(1)));
Dan Gohman4ee29af2009-04-21 02:26:00 +00003322 case Instruction::And:
3323 // For an expression like x&255 that merely masks off the high bits,
3324 // use zext(trunc(x)) as the SCEV expression.
3325 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
Dan Gohman2c73d5f2009-04-25 17:05:40 +00003326 if (CI->isNullValue())
3327 return getSCEV(U->getOperand(1));
Dan Gohmand6c32952009-04-27 01:41:10 +00003328 if (CI->isAllOnesValue())
3329 return getSCEV(U->getOperand(0));
Dan Gohman4ee29af2009-04-21 02:26:00 +00003330 const APInt &A = CI->getValue();
Dan Gohman61ffa8e2009-06-16 19:52:01 +00003331
3332 // Instcombine's ShrinkDemandedConstant may strip bits out of
3333 // constants, obscuring what would otherwise be a low-bits mask.
3334 // Use ComputeMaskedBits to compute what ShrinkDemandedConstant
3335 // knew about to reconstruct a low-bits mask value.
3336 unsigned LZ = A.countLeadingZeros();
3337 unsigned BitWidth = A.getBitWidth();
3338 APInt AllOnes = APInt::getAllOnesValue(BitWidth);
3339 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
3340 ComputeMaskedBits(U->getOperand(0), AllOnes, KnownZero, KnownOne, TD);
3341
3342 APInt EffectiveMask = APInt::getLowBitsSet(BitWidth, BitWidth - LZ);
3343
Dan Gohmanfc3641b2009-06-17 23:54:37 +00003344 if (LZ != 0 && !((~A & ~KnownZero) & EffectiveMask))
Dan Gohman4ee29af2009-04-21 02:26:00 +00003345 return
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003346 getZeroExtendExpr(getTruncateExpr(getSCEV(U->getOperand(0)),
Owen Anderson1d0be152009-08-13 21:58:54 +00003347 IntegerType::get(getContext(), BitWidth - LZ)),
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003348 U->getType());
Dan Gohman4ee29af2009-04-21 02:26:00 +00003349 }
3350 break;
Dan Gohman61ffa8e2009-06-16 19:52:01 +00003351
Dan Gohman6c459a22008-06-22 19:56:46 +00003352 case Instruction::Or:
3353 // If the RHS of the Or is a constant, we may have something like:
3354 // X*4+1 which got turned into X*4|1. Handle this as an Add so loop
3355 // optimizations will transparently handle this case.
3356 //
3357 // In order for this transformation to be safe, the LHS must be of the
3358 // form X*(2^n) and the Or constant must be less than 2^n.
3359 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00003360 const SCEV *LHS = getSCEV(U->getOperand(0));
Dan Gohman6c459a22008-06-22 19:56:46 +00003361 const APInt &CIVal = CI->getValue();
Dan Gohman2c364ad2009-06-19 23:29:04 +00003362 if (GetMinTrailingZeros(LHS) >=
Dan Gohman1f96e672009-09-17 18:05:20 +00003363 (CIVal.getBitWidth() - CIVal.countLeadingZeros())) {
3364 // Build a plain add SCEV.
3365 const SCEV *S = getAddExpr(LHS, getSCEV(CI));
3366 // If the LHS of the add was an addrec and it has no-wrap flags,
3367 // transfer the no-wrap flags, since an or won't introduce a wrap.
3368 if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) {
3369 const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS);
3370 if (OldAR->hasNoUnsignedWrap())
3371 const_cast<SCEVAddRecExpr *>(NewAR)->setHasNoUnsignedWrap(true);
3372 if (OldAR->hasNoSignedWrap())
3373 const_cast<SCEVAddRecExpr *>(NewAR)->setHasNoSignedWrap(true);
3374 }
3375 return S;
3376 }
Chris Lattner53e677a2004-04-02 20:23:17 +00003377 }
Dan Gohman6c459a22008-06-22 19:56:46 +00003378 break;
3379 case Instruction::Xor:
Dan Gohman6c459a22008-06-22 19:56:46 +00003380 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
Nick Lewycky01eaf802008-07-07 06:15:49 +00003381 // If the RHS of the xor is a signbit, then this is just an add.
3382 // Instcombine turns add of signbit into xor as a strength reduction step.
Dan Gohman6c459a22008-06-22 19:56:46 +00003383 if (CI->getValue().isSignBit())
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003384 return getAddExpr(getSCEV(U->getOperand(0)),
3385 getSCEV(U->getOperand(1)));
Nick Lewycky01eaf802008-07-07 06:15:49 +00003386
3387 // If the RHS of xor is -1, then this is a not operation.
Dan Gohman0bac95e2009-05-18 16:17:44 +00003388 if (CI->isAllOnesValue())
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003389 return getNotSCEV(getSCEV(U->getOperand(0)));
Dan Gohman10978bd2009-05-18 16:29:04 +00003390
3391 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
3392 // This is a variant of the check for xor with -1, and it handles
3393 // the case where instcombine has trimmed non-demanded bits out
3394 // of an xor with -1.
3395 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getOperand(0)))
3396 if (ConstantInt *LCI = dyn_cast<ConstantInt>(BO->getOperand(1)))
3397 if (BO->getOpcode() == Instruction::And &&
3398 LCI->getValue() == CI->getValue())
3399 if (const SCEVZeroExtendExpr *Z =
Dan Gohman3034c102009-06-17 01:22:39 +00003400 dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) {
Dan Gohman82052832009-06-18 00:00:20 +00003401 const Type *UTy = U->getType();
Dan Gohman0bba49c2009-07-07 17:06:11 +00003402 const SCEV *Z0 = Z->getOperand();
Dan Gohman82052832009-06-18 00:00:20 +00003403 const Type *Z0Ty = Z0->getType();
3404 unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
3405
Dan Gohman3f46a3a2010-03-01 17:49:51 +00003406 // If C is a low-bits mask, the zero extend is serving to
Dan Gohman82052832009-06-18 00:00:20 +00003407 // mask off the high bits. Complement the operand and
3408 // re-apply the zext.
3409 if (APIntOps::isMask(Z0TySize, CI->getValue()))
3410 return getZeroExtendExpr(getNotSCEV(Z0), UTy);
3411
3412 // If C is a single bit, it may be in the sign-bit position
3413 // before the zero-extend. In this case, represent the xor
3414 // using an add, which is equivalent, and re-apply the zext.
3415 APInt Trunc = APInt(CI->getValue()).trunc(Z0TySize);
3416 if (APInt(Trunc).zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
3417 Trunc.isSignBit())
3418 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
3419 UTy);
Dan Gohman3034c102009-06-17 01:22:39 +00003420 }
Dan Gohman6c459a22008-06-22 19:56:46 +00003421 }
3422 break;
3423
3424 case Instruction::Shl:
3425 // Turn shift left of a constant amount into a multiply.
3426 if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
Dan Gohman4f8eea82010-02-01 18:27:38 +00003427 uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00003428
3429 // If the shift count is not less than the bitwidth, the result of
3430 // the shift is undefined. Don't try to analyze it, because the
3431 // resolution chosen here may differ from the resolution chosen in
3432 // other parts of the compiler.
3433 if (SA->getValue().uge(BitWidth))
3434 break;
3435
Owen Andersoneed707b2009-07-24 23:12:02 +00003436 Constant *X = ConstantInt::get(getContext(),
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00003437 APInt(BitWidth, 1).shl(SA->getZExtValue()));
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003438 return getMulExpr(getSCEV(U->getOperand(0)), getSCEV(X));
Dan Gohman6c459a22008-06-22 19:56:46 +00003439 }
3440 break;
3441
Nick Lewycky01eaf802008-07-07 06:15:49 +00003442 case Instruction::LShr:
Nick Lewycky789558d2009-01-13 09:18:58 +00003443 // Turn logical shift right of a constant into a unsigned divide.
Nick Lewycky01eaf802008-07-07 06:15:49 +00003444 if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
Dan Gohman4f8eea82010-02-01 18:27:38 +00003445 uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00003446
3447 // If the shift count is not less than the bitwidth, the result of
3448 // the shift is undefined. Don't try to analyze it, because the
3449 // resolution chosen here may differ from the resolution chosen in
3450 // other parts of the compiler.
3451 if (SA->getValue().uge(BitWidth))
3452 break;
3453
Owen Andersoneed707b2009-07-24 23:12:02 +00003454 Constant *X = ConstantInt::get(getContext(),
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00003455 APInt(BitWidth, 1).shl(SA->getZExtValue()));
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003456 return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(X));
Nick Lewycky01eaf802008-07-07 06:15:49 +00003457 }
3458 break;
3459
Dan Gohman4ee29af2009-04-21 02:26:00 +00003460 case Instruction::AShr:
3461 // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression.
3462 if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1)))
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00003463 if (Operator *L = dyn_cast<Operator>(U->getOperand(0)))
Dan Gohman4ee29af2009-04-21 02:26:00 +00003464 if (L->getOpcode() == Instruction::Shl &&
3465 L->getOperand(1) == U->getOperand(1)) {
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00003466 uint64_t BitWidth = getTypeSizeInBits(U->getType());
3467
3468 // If the shift count is not less than the bitwidth, the result of
3469 // the shift is undefined. Don't try to analyze it, because the
3470 // resolution chosen here may differ from the resolution chosen in
3471 // other parts of the compiler.
3472 if (CI->getValue().uge(BitWidth))
3473 break;
3474
Dan Gohman2c73d5f2009-04-25 17:05:40 +00003475 uint64_t Amt = BitWidth - CI->getZExtValue();
3476 if (Amt == BitWidth)
3477 return getSCEV(L->getOperand(0)); // shift by zero --> noop
Dan Gohman4ee29af2009-04-21 02:26:00 +00003478 return
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003479 getSignExtendExpr(getTruncateExpr(getSCEV(L->getOperand(0)),
Dan Gohmanddb3eaf2010-04-22 01:35:11 +00003480 IntegerType::get(getContext(),
3481 Amt)),
3482 U->getType());
Dan Gohman4ee29af2009-04-21 02:26:00 +00003483 }
3484 break;
3485
Dan Gohman6c459a22008-06-22 19:56:46 +00003486 case Instruction::Trunc:
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003487 return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
Dan Gohman6c459a22008-06-22 19:56:46 +00003488
3489 case Instruction::ZExt:
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003490 return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
Dan Gohman6c459a22008-06-22 19:56:46 +00003491
3492 case Instruction::SExt:
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003493 return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
Dan Gohman6c459a22008-06-22 19:56:46 +00003494
3495 case Instruction::BitCast:
3496 // BitCasts are no-op casts so we just eliminate the cast.
Dan Gohmanaf79fb52009-04-21 01:07:12 +00003497 if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
Dan Gohman6c459a22008-06-22 19:56:46 +00003498 return getSCEV(U->getOperand(0));
3499 break;
3500
Dan Gohman4f8eea82010-02-01 18:27:38 +00003501 // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can
3502 // lead to pointer expressions which cannot safely be expanded to GEPs,
3503 // because ScalarEvolution doesn't respect the GEP aliasing rules when
3504 // simplifying integer expressions.
Dan Gohman2d1be872009-04-16 03:18:22 +00003505
Dan Gohman26466c02009-05-08 20:26:55 +00003506 case Instruction::GetElementPtr:
Dan Gohmand281ed22009-12-18 02:09:29 +00003507 return createNodeForGEP(cast<GEPOperator>(U));
Dan Gohman2d1be872009-04-16 03:18:22 +00003508
Dan Gohman6c459a22008-06-22 19:56:46 +00003509 case Instruction::PHI:
3510 return createNodeForPHI(cast<PHINode>(U));
3511
3512 case Instruction::Select:
3513 // This could be a smax or umax that was lowered earlier.
3514 // Try to recover it.
3515 if (ICmpInst *ICI = dyn_cast<ICmpInst>(U->getOperand(0))) {
3516 Value *LHS = ICI->getOperand(0);
3517 Value *RHS = ICI->getOperand(1);
3518 switch (ICI->getPredicate()) {
3519 case ICmpInst::ICMP_SLT:
3520 case ICmpInst::ICMP_SLE:
3521 std::swap(LHS, RHS);
3522 // fall through
3523 case ICmpInst::ICMP_SGT:
3524 case ICmpInst::ICMP_SGE:
Dan Gohman9f93d302010-04-24 03:09:42 +00003525 // a >s b ? a+x : b+x -> smax(a, b)+x
3526 // a >s b ? b+x : a+x -> smin(a, b)+x
3527 if (LHS->getType() == U->getType()) {
3528 const SCEV *LS = getSCEV(LHS);
3529 const SCEV *RS = getSCEV(RHS);
3530 const SCEV *LA = getSCEV(U->getOperand(1));
3531 const SCEV *RA = getSCEV(U->getOperand(2));
3532 const SCEV *LDiff = getMinusSCEV(LA, LS);
3533 const SCEV *RDiff = getMinusSCEV(RA, RS);
3534 if (LDiff == RDiff)
3535 return getAddExpr(getSMaxExpr(LS, RS), LDiff);
3536 LDiff = getMinusSCEV(LA, RS);
3537 RDiff = getMinusSCEV(RA, LS);
3538 if (LDiff == RDiff)
3539 return getAddExpr(getSMinExpr(LS, RS), LDiff);
3540 }
Dan Gohman6c459a22008-06-22 19:56:46 +00003541 break;
3542 case ICmpInst::ICMP_ULT:
3543 case ICmpInst::ICMP_ULE:
3544 std::swap(LHS, RHS);
3545 // fall through
3546 case ICmpInst::ICMP_UGT:
3547 case ICmpInst::ICMP_UGE:
Dan Gohman9f93d302010-04-24 03:09:42 +00003548 // a >u b ? a+x : b+x -> umax(a, b)+x
3549 // a >u b ? b+x : a+x -> umin(a, b)+x
3550 if (LHS->getType() == U->getType()) {
3551 const SCEV *LS = getSCEV(LHS);
3552 const SCEV *RS = getSCEV(RHS);
3553 const SCEV *LA = getSCEV(U->getOperand(1));
3554 const SCEV *RA = getSCEV(U->getOperand(2));
3555 const SCEV *LDiff = getMinusSCEV(LA, LS);
3556 const SCEV *RDiff = getMinusSCEV(RA, RS);
3557 if (LDiff == RDiff)
3558 return getAddExpr(getUMaxExpr(LS, RS), LDiff);
3559 LDiff = getMinusSCEV(LA, RS);
3560 RDiff = getMinusSCEV(RA, LS);
3561 if (LDiff == RDiff)
3562 return getAddExpr(getUMinExpr(LS, RS), LDiff);
3563 }
Dan Gohman6c459a22008-06-22 19:56:46 +00003564 break;
Dan Gohman30fb5122009-06-18 20:21:07 +00003565 case ICmpInst::ICMP_NE:
Dan Gohman9f93d302010-04-24 03:09:42 +00003566 // n != 0 ? n+x : 1+x -> umax(n, 1)+x
3567 if (LHS->getType() == U->getType() &&
Dan Gohman30fb5122009-06-18 20:21:07 +00003568 isa<ConstantInt>(RHS) &&
Dan Gohman9f93d302010-04-24 03:09:42 +00003569 cast<ConstantInt>(RHS)->isZero()) {
3570 const SCEV *One = getConstant(LHS->getType(), 1);
3571 const SCEV *LS = getSCEV(LHS);
3572 const SCEV *LA = getSCEV(U->getOperand(1));
3573 const SCEV *RA = getSCEV(U->getOperand(2));
3574 const SCEV *LDiff = getMinusSCEV(LA, LS);
3575 const SCEV *RDiff = getMinusSCEV(RA, One);
3576 if (LDiff == RDiff)
Dan Gohman58a85b92010-08-13 20:17:14 +00003577 return getAddExpr(getUMaxExpr(One, LS), LDiff);
Dan Gohman9f93d302010-04-24 03:09:42 +00003578 }
Dan Gohman30fb5122009-06-18 20:21:07 +00003579 break;
3580 case ICmpInst::ICMP_EQ:
Dan Gohman9f93d302010-04-24 03:09:42 +00003581 // n == 0 ? 1+x : n+x -> umax(n, 1)+x
3582 if (LHS->getType() == U->getType() &&
Dan Gohman30fb5122009-06-18 20:21:07 +00003583 isa<ConstantInt>(RHS) &&
Dan Gohman9f93d302010-04-24 03:09:42 +00003584 cast<ConstantInt>(RHS)->isZero()) {
3585 const SCEV *One = getConstant(LHS->getType(), 1);
3586 const SCEV *LS = getSCEV(LHS);
3587 const SCEV *LA = getSCEV(U->getOperand(1));
3588 const SCEV *RA = getSCEV(U->getOperand(2));
3589 const SCEV *LDiff = getMinusSCEV(LA, One);
3590 const SCEV *RDiff = getMinusSCEV(RA, LS);
3591 if (LDiff == RDiff)
Dan Gohman58a85b92010-08-13 20:17:14 +00003592 return getAddExpr(getUMaxExpr(One, LS), LDiff);
Dan Gohman9f93d302010-04-24 03:09:42 +00003593 }
Dan Gohman30fb5122009-06-18 20:21:07 +00003594 break;
Dan Gohman6c459a22008-06-22 19:56:46 +00003595 default:
3596 break;
3597 }
3598 }
3599
3600 default: // We cannot analyze this expression.
3601 break;
Chris Lattner53e677a2004-04-02 20:23:17 +00003602 }
3603
Dan Gohmanf8a8be82009-04-21 23:15:49 +00003604 return getUnknown(V);
Chris Lattner53e677a2004-04-02 20:23:17 +00003605}
3606
3607
3608
3609//===----------------------------------------------------------------------===//
3610// Iteration Count Computation Code
3611//
3612
Dan Gohman46bdfb02009-02-24 18:55:53 +00003613/// getBackedgeTakenCount - If the specified loop has a predictable
3614/// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute
3615/// object. The backedge-taken count is the number of times the loop header
3616/// will be branched to from within the loop. This is one less than the
3617/// trip count of the loop, since it doesn't count the first iteration,
3618/// when the header is branched to from outside the loop.
3619///
3620/// Note that it is not valid to call this method on a loop without a
3621/// loop-invariant backedge-taken count (see
3622/// hasLoopInvariantBackedgeTakenCount).
3623///
Dan Gohman0bba49c2009-07-07 17:06:11 +00003624const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) {
Dan Gohmana1af7572009-04-30 20:47:05 +00003625 return getBackedgeTakenInfo(L).Exact;
3626}
3627
3628/// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except
3629/// return the least SCEV value that is known never to be less than the
3630/// actual backedge taken count.
Dan Gohman0bba49c2009-07-07 17:06:11 +00003631const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) {
Dan Gohmana1af7572009-04-30 20:47:05 +00003632 return getBackedgeTakenInfo(L).Max;
3633}
3634
Dan Gohman59ae6b92009-07-08 19:23:34 +00003635/// PushLoopPHIs - Push PHI nodes in the header of the given loop
3636/// onto the given Worklist.
3637static void
3638PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) {
3639 BasicBlock *Header = L->getHeader();
3640
3641 // Push all Loop-header PHIs onto the Worklist stack.
3642 for (BasicBlock::iterator I = Header->begin();
3643 PHINode *PN = dyn_cast<PHINode>(I); ++I)
3644 Worklist.push_back(PN);
3645}
3646
Dan Gohmana1af7572009-04-30 20:47:05 +00003647const ScalarEvolution::BackedgeTakenInfo &
3648ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
Dan Gohman01ecca22009-04-27 20:16:15 +00003649 // Initially insert a CouldNotCompute for this loop. If the insertion
Dan Gohman3f46a3a2010-03-01 17:49:51 +00003650 // succeeds, proceed to actually compute a backedge-taken count and
Dan Gohman01ecca22009-04-27 20:16:15 +00003651 // update the value. The temporary CouldNotCompute value tells SCEV
3652 // code elsewhere that it shouldn't attempt to request a new
3653 // backedge-taken count, which could result in infinite recursion.
Dan Gohman5d984912009-12-18 01:14:11 +00003654 std::pair<std::map<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
Dan Gohman01ecca22009-04-27 20:16:15 +00003655 BackedgeTakenCounts.insert(std::make_pair(L, getCouldNotCompute()));
3656 if (Pair.second) {
Dan Gohman93dacad2010-01-26 16:46:18 +00003657 BackedgeTakenInfo BECount = ComputeBackedgeTakenCount(L);
3658 if (BECount.Exact != getCouldNotCompute()) {
3659 assert(BECount.Exact->isLoopInvariant(L) &&
3660 BECount.Max->isLoopInvariant(L) &&
3661 "Computed backedge-taken count isn't loop invariant for loop!");
Chris Lattner53e677a2004-04-02 20:23:17 +00003662 ++NumTripCountsComputed;
Dan Gohman01ecca22009-04-27 20:16:15 +00003663
Dan Gohman01ecca22009-04-27 20:16:15 +00003664 // Update the value in the map.
Dan Gohman93dacad2010-01-26 16:46:18 +00003665 Pair.first->second = BECount;
Dan Gohmana334aa72009-06-22 00:31:57 +00003666 } else {
Dan Gohman93dacad2010-01-26 16:46:18 +00003667 if (BECount.Max != getCouldNotCompute())
Dan Gohmana334aa72009-06-22 00:31:57 +00003668 // Update the value in the map.
Dan Gohman93dacad2010-01-26 16:46:18 +00003669 Pair.first->second = BECount;
Dan Gohmana334aa72009-06-22 00:31:57 +00003670 if (isa<PHINode>(L->getHeader()->begin()))
3671 // Only count loops that have phi nodes as not being computable.
3672 ++NumTripCountsNotComputed;
Chris Lattner53e677a2004-04-02 20:23:17 +00003673 }
Dan Gohmana1af7572009-04-30 20:47:05 +00003674
3675 // Now that we know more about the trip count for this loop, forget any
3676 // existing SCEV values for PHI nodes in this loop since they are only
Dan Gohman59ae6b92009-07-08 19:23:34 +00003677 // conservative estimates made without the benefit of trip count
Dan Gohman4c7279a2009-10-31 15:04:55 +00003678 // information. This is similar to the code in forgetLoop, except that
3679 // it handles SCEVUnknown PHI nodes specially.
Dan Gohman93dacad2010-01-26 16:46:18 +00003680 if (BECount.hasAnyInfo()) {
Dan Gohman59ae6b92009-07-08 19:23:34 +00003681 SmallVector<Instruction *, 16> Worklist;
3682 PushLoopPHIs(L, Worklist);
3683
3684 SmallPtrSet<Instruction *, 8> Visited;
3685 while (!Worklist.empty()) {
3686 Instruction *I = Worklist.pop_back_val();
3687 if (!Visited.insert(I)) continue;
3688
Dan Gohman5d984912009-12-18 01:14:11 +00003689 std::map<SCEVCallbackVH, const SCEV *>::iterator It =
Dan Gohman59ae6b92009-07-08 19:23:34 +00003690 Scalars.find(static_cast<Value *>(I));
3691 if (It != Scalars.end()) {
3692 // SCEVUnknown for a PHI either means that it has an unrecognized
3693 // structure, or it's a PHI that's in the progress of being computed
Dan Gohmanba701882009-07-13 22:04:06 +00003694 // by createNodeForPHI. In the former case, additional loop trip
3695 // count information isn't going to change anything. In the later
3696 // case, createNodeForPHI will perform the necessary updates on its
3697 // own when it gets to that point.
Dan Gohman42214892009-08-31 21:15:23 +00003698 if (!isa<PHINode>(I) || !isa<SCEVUnknown>(It->second)) {
3699 ValuesAtScopes.erase(It->second);
Dan Gohman59ae6b92009-07-08 19:23:34 +00003700 Scalars.erase(It);
Dan Gohman42214892009-08-31 21:15:23 +00003701 }
Dan Gohman59ae6b92009-07-08 19:23:34 +00003702 if (PHINode *PN = dyn_cast<PHINode>(I))
3703 ConstantEvolutionLoopExitValue.erase(PN);
3704 }
3705
3706 PushDefUseChildren(I, Worklist);
3707 }
3708 }
Chris Lattner53e677a2004-04-02 20:23:17 +00003709 }
Dan Gohman01ecca22009-04-27 20:16:15 +00003710 return Pair.first->second;
Chris Lattner53e677a2004-04-02 20:23:17 +00003711}
3712
Dan Gohman4c7279a2009-10-31 15:04:55 +00003713/// forgetLoop - This method should be called by the client when it has
3714/// changed a loop in a way that may effect ScalarEvolution's ability to
3715/// compute a trip count, or if the loop is deleted.
3716void ScalarEvolution::forgetLoop(const Loop *L) {
3717 // Drop any stored trip count value.
Dan Gohman46bdfb02009-02-24 18:55:53 +00003718 BackedgeTakenCounts.erase(L);
Dan Gohmanfb7d35f2009-05-02 17:43:35 +00003719
Dan Gohman4c7279a2009-10-31 15:04:55 +00003720 // Drop information about expressions based on loop-header PHIs.
Dan Gohman35738ac2009-05-04 22:30:44 +00003721 SmallVector<Instruction *, 16> Worklist;
Dan Gohman59ae6b92009-07-08 19:23:34 +00003722 PushLoopPHIs(L, Worklist);
Dan Gohman35738ac2009-05-04 22:30:44 +00003723
Dan Gohman59ae6b92009-07-08 19:23:34 +00003724 SmallPtrSet<Instruction *, 8> Visited;
Dan Gohman35738ac2009-05-04 22:30:44 +00003725 while (!Worklist.empty()) {
3726 Instruction *I = Worklist.pop_back_val();
Dan Gohman59ae6b92009-07-08 19:23:34 +00003727 if (!Visited.insert(I)) continue;
3728
Dan Gohman5d984912009-12-18 01:14:11 +00003729 std::map<SCEVCallbackVH, const SCEV *>::iterator It =
Dan Gohman59ae6b92009-07-08 19:23:34 +00003730 Scalars.find(static_cast<Value *>(I));
3731 if (It != Scalars.end()) {
Dan Gohman42214892009-08-31 21:15:23 +00003732 ValuesAtScopes.erase(It->second);
Dan Gohman59ae6b92009-07-08 19:23:34 +00003733 Scalars.erase(It);
Dan Gohman59ae6b92009-07-08 19:23:34 +00003734 if (PHINode *PN = dyn_cast<PHINode>(I))
3735 ConstantEvolutionLoopExitValue.erase(PN);
3736 }
3737
3738 PushDefUseChildren(I, Worklist);
Dan Gohman35738ac2009-05-04 22:30:44 +00003739 }
Dan Gohman60f8a632009-02-17 20:49:49 +00003740}
3741
Eric Christophere6cbfa62010-07-29 01:25:38 +00003742/// forgetValue - This method should be called by the client when it has
3743/// changed a value in a way that may effect its value, or which may
3744/// disconnect it from a def-use chain linking it to a loop.
3745void ScalarEvolution::forgetValue(Value *V) {
Dale Johannesen45a2d7d2010-02-19 07:14:22 +00003746 Instruction *I = dyn_cast<Instruction>(V);
3747 if (!I) return;
3748
3749 // Drop information about expressions based on loop-header PHIs.
3750 SmallVector<Instruction *, 16> Worklist;
3751 Worklist.push_back(I);
3752
3753 SmallPtrSet<Instruction *, 8> Visited;
3754 while (!Worklist.empty()) {
3755 I = Worklist.pop_back_val();
3756 if (!Visited.insert(I)) continue;
3757
3758 std::map<SCEVCallbackVH, const SCEV *>::iterator It =
3759 Scalars.find(static_cast<Value *>(I));
3760 if (It != Scalars.end()) {
3761 ValuesAtScopes.erase(It->second);
3762 Scalars.erase(It);
3763 if (PHINode *PN = dyn_cast<PHINode>(I))
3764 ConstantEvolutionLoopExitValue.erase(PN);
3765 }
3766
3767 PushDefUseChildren(I, Worklist);
3768 }
3769}
3770
Dan Gohman46bdfb02009-02-24 18:55:53 +00003771/// ComputeBackedgeTakenCount - Compute the number of times the backedge
3772/// of the specified loop will execute.
Dan Gohmana1af7572009-04-30 20:47:05 +00003773ScalarEvolution::BackedgeTakenInfo
3774ScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) {
Dan Gohman5d984912009-12-18 01:14:11 +00003775 SmallVector<BasicBlock *, 8> ExitingBlocks;
Dan Gohmana334aa72009-06-22 00:31:57 +00003776 L->getExitingBlocks(ExitingBlocks);
Chris Lattner53e677a2004-04-02 20:23:17 +00003777
Dan Gohmana334aa72009-06-22 00:31:57 +00003778 // Examine all exits and pick the most conservative values.
Dan Gohman0bba49c2009-07-07 17:06:11 +00003779 const SCEV *BECount = getCouldNotCompute();
3780 const SCEV *MaxBECount = getCouldNotCompute();
Dan Gohmana334aa72009-06-22 00:31:57 +00003781 bool CouldNotComputeBECount = false;
Dan Gohmana334aa72009-06-22 00:31:57 +00003782 for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
3783 BackedgeTakenInfo NewBTI =
3784 ComputeBackedgeTakenCountFromExit(L, ExitingBlocks[i]);
Chris Lattner53e677a2004-04-02 20:23:17 +00003785
Dan Gohman1c343752009-06-27 21:21:31 +00003786 if (NewBTI.Exact == getCouldNotCompute()) {
Dan Gohmana334aa72009-06-22 00:31:57 +00003787 // We couldn't compute an exact value for this exit, so
Dan Gohmand32f5bf2009-06-22 21:10:22 +00003788 // we won't be able to compute an exact value for the loop.
Dan Gohmana334aa72009-06-22 00:31:57 +00003789 CouldNotComputeBECount = true;
Dan Gohman1c343752009-06-27 21:21:31 +00003790 BECount = getCouldNotCompute();
Dan Gohmana334aa72009-06-22 00:31:57 +00003791 } else if (!CouldNotComputeBECount) {
Dan Gohman1c343752009-06-27 21:21:31 +00003792 if (BECount == getCouldNotCompute())
Dan Gohmana334aa72009-06-22 00:31:57 +00003793 BECount = NewBTI.Exact;
Dan Gohmana334aa72009-06-22 00:31:57 +00003794 else
Dan Gohman40a5a1b2009-06-24 01:18:18 +00003795 BECount = getUMinFromMismatchedTypes(BECount, NewBTI.Exact);
Dan Gohmana334aa72009-06-22 00:31:57 +00003796 }
Dan Gohman1c343752009-06-27 21:21:31 +00003797 if (MaxBECount == getCouldNotCompute())
Dan Gohman40a5a1b2009-06-24 01:18:18 +00003798 MaxBECount = NewBTI.Max;
Dan Gohman1c343752009-06-27 21:21:31 +00003799 else if (NewBTI.Max != getCouldNotCompute())
Dan Gohman40a5a1b2009-06-24 01:18:18 +00003800 MaxBECount = getUMinFromMismatchedTypes(MaxBECount, NewBTI.Max);
Dan Gohmana334aa72009-06-22 00:31:57 +00003801 }
3802
3803 return BackedgeTakenInfo(BECount, MaxBECount);
3804}
3805
3806/// ComputeBackedgeTakenCountFromExit - Compute the number of times the backedge
3807/// of the specified loop will execute if it exits via the specified block.
3808ScalarEvolution::BackedgeTakenInfo
3809ScalarEvolution::ComputeBackedgeTakenCountFromExit(const Loop *L,
3810 BasicBlock *ExitingBlock) {
3811
3812 // Okay, we've chosen an exiting block. See what condition causes us to
3813 // exit at this block.
Chris Lattner53e677a2004-04-02 20:23:17 +00003814 //
3815 // FIXME: we should be able to handle switch instructions (with a single exit)
Chris Lattner53e677a2004-04-02 20:23:17 +00003816 BranchInst *ExitBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator());
Dan Gohman1c343752009-06-27 21:21:31 +00003817 if (ExitBr == 0) return getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00003818 assert(ExitBr->isConditional() && "If unconditional, it can't be in loop!");
Dan Gohman64a845e2009-06-24 04:48:43 +00003819
Chris Lattner8b0e3602007-01-07 02:24:26 +00003820 // At this point, we know we have a conditional branch that determines whether
3821 // the loop is exited. However, we don't know if the branch is executed each
3822 // time through the loop. If not, then the execution count of the branch will
3823 // not be equal to the trip count of the loop.
3824 //
3825 // Currently we check for this by checking to see if the Exit branch goes to
3826 // the loop header. If so, we know it will always execute the same number of
Chris Lattner192e4032007-01-14 01:24:47 +00003827 // times as the loop. We also handle the case where the exit block *is* the
Dan Gohmana334aa72009-06-22 00:31:57 +00003828 // loop header. This is common for un-rotated loops.
3829 //
3830 // If both of those tests fail, walk up the unique predecessor chain to the
3831 // header, stopping if there is an edge that doesn't exit the loop. If the
3832 // header is reached, the execution count of the branch will be equal to the
3833 // trip count of the loop.
3834 //
3835 // More extensive analysis could be done to handle more cases here.
3836 //
Chris Lattner8b0e3602007-01-07 02:24:26 +00003837 if (ExitBr->getSuccessor(0) != L->getHeader() &&
Chris Lattner192e4032007-01-14 01:24:47 +00003838 ExitBr->getSuccessor(1) != L->getHeader() &&
Dan Gohmana334aa72009-06-22 00:31:57 +00003839 ExitBr->getParent() != L->getHeader()) {
3840 // The simple checks failed, try climbing the unique predecessor chain
3841 // up to the header.
3842 bool Ok = false;
3843 for (BasicBlock *BB = ExitBr->getParent(); BB; ) {
3844 BasicBlock *Pred = BB->getUniquePredecessor();
3845 if (!Pred)
Dan Gohman1c343752009-06-27 21:21:31 +00003846 return getCouldNotCompute();
Dan Gohmana334aa72009-06-22 00:31:57 +00003847 TerminatorInst *PredTerm = Pred->getTerminator();
3848 for (unsigned i = 0, e = PredTerm->getNumSuccessors(); i != e; ++i) {
3849 BasicBlock *PredSucc = PredTerm->getSuccessor(i);
3850 if (PredSucc == BB)
3851 continue;
3852 // If the predecessor has a successor that isn't BB and isn't
3853 // outside the loop, assume the worst.
3854 if (L->contains(PredSucc))
Dan Gohman1c343752009-06-27 21:21:31 +00003855 return getCouldNotCompute();
Dan Gohmana334aa72009-06-22 00:31:57 +00003856 }
3857 if (Pred == L->getHeader()) {
3858 Ok = true;
3859 break;
3860 }
3861 BB = Pred;
3862 }
3863 if (!Ok)
Dan Gohman1c343752009-06-27 21:21:31 +00003864 return getCouldNotCompute();
Dan Gohmana334aa72009-06-22 00:31:57 +00003865 }
3866
Dan Gohman3f46a3a2010-03-01 17:49:51 +00003867 // Proceed to the next level to examine the exit condition expression.
Dan Gohmana334aa72009-06-22 00:31:57 +00003868 return ComputeBackedgeTakenCountFromExitCond(L, ExitBr->getCondition(),
3869 ExitBr->getSuccessor(0),
3870 ExitBr->getSuccessor(1));
3871}
3872
3873/// ComputeBackedgeTakenCountFromExitCond - Compute the number of times the
3874/// backedge of the specified loop will execute if its exit condition
3875/// were a conditional branch of ExitCond, TBB, and FBB.
3876ScalarEvolution::BackedgeTakenInfo
3877ScalarEvolution::ComputeBackedgeTakenCountFromExitCond(const Loop *L,
3878 Value *ExitCond,
3879 BasicBlock *TBB,
3880 BasicBlock *FBB) {
Dan Gohman40a5a1b2009-06-24 01:18:18 +00003881 // Check if the controlling expression for this loop is an And or Or.
Dan Gohmana334aa72009-06-22 00:31:57 +00003882 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) {
3883 if (BO->getOpcode() == Instruction::And) {
3884 // Recurse on the operands of the and.
3885 BackedgeTakenInfo BTI0 =
3886 ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(0), TBB, FBB);
3887 BackedgeTakenInfo BTI1 =
3888 ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(1), TBB, FBB);
Dan Gohman0bba49c2009-07-07 17:06:11 +00003889 const SCEV *BECount = getCouldNotCompute();
3890 const SCEV *MaxBECount = getCouldNotCompute();
Dan Gohmana334aa72009-06-22 00:31:57 +00003891 if (L->contains(TBB)) {
3892 // Both conditions must be true for the loop to continue executing.
3893 // Choose the less conservative count.
Dan Gohman1c343752009-06-27 21:21:31 +00003894 if (BTI0.Exact == getCouldNotCompute() ||
3895 BTI1.Exact == getCouldNotCompute())
3896 BECount = getCouldNotCompute();
Dan Gohman60e9b072009-06-22 15:09:28 +00003897 else
3898 BECount = getUMinFromMismatchedTypes(BTI0.Exact, BTI1.Exact);
Dan Gohman1c343752009-06-27 21:21:31 +00003899 if (BTI0.Max == getCouldNotCompute())
Dan Gohmana334aa72009-06-22 00:31:57 +00003900 MaxBECount = BTI1.Max;
Dan Gohman1c343752009-06-27 21:21:31 +00003901 else if (BTI1.Max == getCouldNotCompute())
Dan Gohmana334aa72009-06-22 00:31:57 +00003902 MaxBECount = BTI0.Max;
Dan Gohman60e9b072009-06-22 15:09:28 +00003903 else
3904 MaxBECount = getUMinFromMismatchedTypes(BTI0.Max, BTI1.Max);
Dan Gohmana334aa72009-06-22 00:31:57 +00003905 } else {
Dan Gohman4ee87392010-08-11 00:12:36 +00003906 // Both conditions must be true at the same time for the loop to exit.
3907 // For now, be conservative.
Dan Gohmana334aa72009-06-22 00:31:57 +00003908 assert(L->contains(FBB) && "Loop block has no successor in loop!");
Dan Gohman4ee87392010-08-11 00:12:36 +00003909 if (BTI0.Max == BTI1.Max)
3910 MaxBECount = BTI0.Max;
3911 if (BTI0.Exact == BTI1.Exact)
3912 BECount = BTI0.Exact;
Dan Gohmana334aa72009-06-22 00:31:57 +00003913 }
3914
3915 return BackedgeTakenInfo(BECount, MaxBECount);
3916 }
3917 if (BO->getOpcode() == Instruction::Or) {
3918 // Recurse on the operands of the or.
3919 BackedgeTakenInfo BTI0 =
3920 ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(0), TBB, FBB);
3921 BackedgeTakenInfo BTI1 =
3922 ComputeBackedgeTakenCountFromExitCond(L, BO->getOperand(1), TBB, FBB);
Dan Gohman0bba49c2009-07-07 17:06:11 +00003923 const SCEV *BECount = getCouldNotCompute();
3924 const SCEV *MaxBECount = getCouldNotCompute();
Dan Gohmana334aa72009-06-22 00:31:57 +00003925 if (L->contains(FBB)) {
3926 // Both conditions must be false for the loop to continue executing.
3927 // Choose the less conservative count.
Dan Gohman1c343752009-06-27 21:21:31 +00003928 if (BTI0.Exact == getCouldNotCompute() ||
3929 BTI1.Exact == getCouldNotCompute())
3930 BECount = getCouldNotCompute();
Dan Gohman60e9b072009-06-22 15:09:28 +00003931 else
3932 BECount = getUMinFromMismatchedTypes(BTI0.Exact, BTI1.Exact);
Dan Gohman1c343752009-06-27 21:21:31 +00003933 if (BTI0.Max == getCouldNotCompute())
Dan Gohmana334aa72009-06-22 00:31:57 +00003934 MaxBECount = BTI1.Max;
Dan Gohman1c343752009-06-27 21:21:31 +00003935 else if (BTI1.Max == getCouldNotCompute())
Dan Gohmana334aa72009-06-22 00:31:57 +00003936 MaxBECount = BTI0.Max;
Dan Gohman60e9b072009-06-22 15:09:28 +00003937 else
3938 MaxBECount = getUMinFromMismatchedTypes(BTI0.Max, BTI1.Max);
Dan Gohmana334aa72009-06-22 00:31:57 +00003939 } else {
Dan Gohman4ee87392010-08-11 00:12:36 +00003940 // Both conditions must be false at the same time for the loop to exit.
3941 // For now, be conservative.
Dan Gohmana334aa72009-06-22 00:31:57 +00003942 assert(L->contains(TBB) && "Loop block has no successor in loop!");
Dan Gohman4ee87392010-08-11 00:12:36 +00003943 if (BTI0.Max == BTI1.Max)
3944 MaxBECount = BTI0.Max;
3945 if (BTI0.Exact == BTI1.Exact)
3946 BECount = BTI0.Exact;
Dan Gohmana334aa72009-06-22 00:31:57 +00003947 }
3948
3949 return BackedgeTakenInfo(BECount, MaxBECount);
3950 }
3951 }
3952
3953 // With an icmp, it may be feasible to compute an exact backedge-taken count.
Dan Gohman3f46a3a2010-03-01 17:49:51 +00003954 // Proceed to the next level to examine the icmp.
Dan Gohmana334aa72009-06-22 00:31:57 +00003955 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond))
3956 return ComputeBackedgeTakenCountFromExitCondICmp(L, ExitCondICmp, TBB, FBB);
Reid Spencere4d87aa2006-12-23 06:05:41 +00003957
Dan Gohman00cb5b72010-02-19 18:12:07 +00003958 // Check for a constant condition. These are normally stripped out by
3959 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
3960 // preserve the CFG and is temporarily leaving constant conditions
3961 // in place.
3962 if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
3963 if (L->contains(FBB) == !CI->getZExtValue())
3964 // The backedge is always taken.
3965 return getCouldNotCompute();
3966 else
3967 // The backedge is never taken.
Dan Gohmandeff6212010-05-03 22:09:21 +00003968 return getConstant(CI->getType(), 0);
Dan Gohman00cb5b72010-02-19 18:12:07 +00003969 }
3970
Eli Friedman361e54d2009-05-09 12:32:42 +00003971 // If it's not an integer or pointer comparison then compute it the hard way.
Dan Gohmana334aa72009-06-22 00:31:57 +00003972 return ComputeBackedgeTakenCountExhaustively(L, ExitCond, !L->contains(TBB));
3973}
3974
3975/// ComputeBackedgeTakenCountFromExitCondICmp - Compute the number of times the
3976/// backedge of the specified loop will execute if its exit condition
3977/// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB.
3978ScalarEvolution::BackedgeTakenInfo
3979ScalarEvolution::ComputeBackedgeTakenCountFromExitCondICmp(const Loop *L,
3980 ICmpInst *ExitCond,
3981 BasicBlock *TBB,
3982 BasicBlock *FBB) {
Chris Lattner53e677a2004-04-02 20:23:17 +00003983
Reid Spencere4d87aa2006-12-23 06:05:41 +00003984 // If the condition was exit on true, convert the condition to exit on false
3985 ICmpInst::Predicate Cond;
Dan Gohmana334aa72009-06-22 00:31:57 +00003986 if (!L->contains(FBB))
Reid Spencere4d87aa2006-12-23 06:05:41 +00003987 Cond = ExitCond->getPredicate();
Chris Lattner673e02b2004-10-12 01:49:27 +00003988 else
Reid Spencere4d87aa2006-12-23 06:05:41 +00003989 Cond = ExitCond->getInversePredicate();
Chris Lattner673e02b2004-10-12 01:49:27 +00003990
3991 // Handle common loops like: for (X = "string"; *X; ++X)
3992 if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0)))
3993 if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) {
Dan Gohmanf6d009f2010-02-24 17:31:30 +00003994 BackedgeTakenInfo ItCnt =
Dan Gohman46bdfb02009-02-24 18:55:53 +00003995 ComputeLoadConstantCompareBackedgeTakenCount(LI, RHS, L, Cond);
Dan Gohmanf6d009f2010-02-24 17:31:30 +00003996 if (ItCnt.hasAnyInfo())
3997 return ItCnt;
Chris Lattner673e02b2004-10-12 01:49:27 +00003998 }
3999
Dan Gohman0bba49c2009-07-07 17:06:11 +00004000 const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
4001 const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
Chris Lattner53e677a2004-04-02 20:23:17 +00004002
4003 // Try to evaluate any dependencies out of the loop.
Dan Gohmand594e6f2009-05-24 23:25:42 +00004004 LHS = getSCEVAtScope(LHS, L);
4005 RHS = getSCEVAtScope(RHS, L);
Chris Lattner53e677a2004-04-02 20:23:17 +00004006
Dan Gohman64a845e2009-06-24 04:48:43 +00004007 // At this point, we would like to compute how many iterations of the
Reid Spencere4d87aa2006-12-23 06:05:41 +00004008 // loop the predicate will return true for these inputs.
Dan Gohman70ff4cf2008-09-16 18:52:57 +00004009 if (LHS->isLoopInvariant(L) && !RHS->isLoopInvariant(L)) {
4010 // If there is a loop-invariant, force it into the RHS.
Chris Lattner53e677a2004-04-02 20:23:17 +00004011 std::swap(LHS, RHS);
Reid Spencere4d87aa2006-12-23 06:05:41 +00004012 Cond = ICmpInst::getSwappedPredicate(Cond);
Chris Lattner53e677a2004-04-02 20:23:17 +00004013 }
4014
Dan Gohman03557dc2010-05-03 16:35:17 +00004015 // Simplify the operands before analyzing them.
4016 (void)SimplifyICmpOperands(Cond, LHS, RHS);
4017
Chris Lattner53e677a2004-04-02 20:23:17 +00004018 // If we have a comparison of a chrec against a constant, try to use value
4019 // ranges to answer this query.
Dan Gohman622ed672009-05-04 22:02:23 +00004020 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
4021 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
Chris Lattner53e677a2004-04-02 20:23:17 +00004022 if (AddRec->getLoop() == L) {
Eli Friedman361e54d2009-05-09 12:32:42 +00004023 // Form the constant range.
4024 ConstantRange CompRange(
4025 ICmpInst::makeConstantRange(Cond, RHSC->getValue()->getValue()));
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004026
Dan Gohman0bba49c2009-07-07 17:06:11 +00004027 const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this);
Eli Friedman361e54d2009-05-09 12:32:42 +00004028 if (!isa<SCEVCouldNotCompute>(Ret)) return Ret;
Chris Lattner53e677a2004-04-02 20:23:17 +00004029 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004030
Chris Lattner53e677a2004-04-02 20:23:17 +00004031 switch (Cond) {
Reid Spencere4d87aa2006-12-23 06:05:41 +00004032 case ICmpInst::ICMP_NE: { // while (X != Y)
Chris Lattner53e677a2004-04-02 20:23:17 +00004033 // Convert to: while (X-Y != 0)
Dan Gohmanf6d009f2010-02-24 17:31:30 +00004034 BackedgeTakenInfo BTI = HowFarToZero(getMinusSCEV(LHS, RHS), L);
4035 if (BTI.hasAnyInfo()) return BTI;
Chris Lattner53e677a2004-04-02 20:23:17 +00004036 break;
Reid Spencere4d87aa2006-12-23 06:05:41 +00004037 }
Dan Gohman4c0d5d52009-08-20 16:42:55 +00004038 case ICmpInst::ICMP_EQ: { // while (X == Y)
4039 // Convert to: while (X-Y == 0)
Dan Gohmanf6d009f2010-02-24 17:31:30 +00004040 BackedgeTakenInfo BTI = HowFarToNonZero(getMinusSCEV(LHS, RHS), L);
4041 if (BTI.hasAnyInfo()) return BTI;
Chris Lattner53e677a2004-04-02 20:23:17 +00004042 break;
Reid Spencere4d87aa2006-12-23 06:05:41 +00004043 }
4044 case ICmpInst::ICMP_SLT: {
Dan Gohmana1af7572009-04-30 20:47:05 +00004045 BackedgeTakenInfo BTI = HowManyLessThans(LHS, RHS, L, true);
4046 if (BTI.hasAnyInfo()) return BTI;
Chris Lattnerdb25de42005-08-15 23:33:51 +00004047 break;
Reid Spencere4d87aa2006-12-23 06:05:41 +00004048 }
4049 case ICmpInst::ICMP_SGT: {
Dan Gohmana1af7572009-04-30 20:47:05 +00004050 BackedgeTakenInfo BTI = HowManyLessThans(getNotSCEV(LHS),
4051 getNotSCEV(RHS), L, true);
4052 if (BTI.hasAnyInfo()) return BTI;
Nick Lewyckyd6dac0e2007-08-06 19:21:00 +00004053 break;
4054 }
4055 case ICmpInst::ICMP_ULT: {
Dan Gohmana1af7572009-04-30 20:47:05 +00004056 BackedgeTakenInfo BTI = HowManyLessThans(LHS, RHS, L, false);
4057 if (BTI.hasAnyInfo()) return BTI;
Nick Lewyckyd6dac0e2007-08-06 19:21:00 +00004058 break;
4059 }
4060 case ICmpInst::ICMP_UGT: {
Dan Gohmana1af7572009-04-30 20:47:05 +00004061 BackedgeTakenInfo BTI = HowManyLessThans(getNotSCEV(LHS),
4062 getNotSCEV(RHS), L, false);
4063 if (BTI.hasAnyInfo()) return BTI;
Chris Lattnerdb25de42005-08-15 23:33:51 +00004064 break;
Reid Spencere4d87aa2006-12-23 06:05:41 +00004065 }
Chris Lattner53e677a2004-04-02 20:23:17 +00004066 default:
Chris Lattnerd18d9dc2004-04-02 20:26:46 +00004067#if 0
David Greene25e0e872009-12-23 22:18:14 +00004068 dbgs() << "ComputeBackedgeTakenCount ";
Chris Lattner53e677a2004-04-02 20:23:17 +00004069 if (ExitCond->getOperand(0)->getType()->isUnsigned())
David Greene25e0e872009-12-23 22:18:14 +00004070 dbgs() << "[unsigned] ";
4071 dbgs() << *LHS << " "
Dan Gohman64a845e2009-06-24 04:48:43 +00004072 << Instruction::getOpcodeName(Instruction::ICmp)
Reid Spencere4d87aa2006-12-23 06:05:41 +00004073 << " " << *RHS << "\n";
Chris Lattnerd18d9dc2004-04-02 20:26:46 +00004074#endif
Chris Lattnere34c0b42004-04-03 00:43:03 +00004075 break;
Chris Lattner53e677a2004-04-02 20:23:17 +00004076 }
Dan Gohman46bdfb02009-02-24 18:55:53 +00004077 return
Dan Gohmana334aa72009-06-22 00:31:57 +00004078 ComputeBackedgeTakenCountExhaustively(L, ExitCond, !L->contains(TBB));
Chris Lattner7980fb92004-04-17 18:36:24 +00004079}
4080
Chris Lattner673e02b2004-10-12 01:49:27 +00004081static ConstantInt *
Dan Gohman246b2562007-10-22 18:31:58 +00004082EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
4083 ScalarEvolution &SE) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00004084 const SCEV *InVal = SE.getConstant(C);
4085 const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE);
Chris Lattner673e02b2004-10-12 01:49:27 +00004086 assert(isa<SCEVConstant>(Val) &&
4087 "Evaluation of SCEV at constant didn't fold correctly?");
4088 return cast<SCEVConstant>(Val)->getValue();
4089}
4090
4091/// GetAddressedElementFromGlobal - Given a global variable with an initializer
4092/// and a GEP expression (missing the pointer index) indexing into it, return
4093/// the addressed element of the initializer or null if the index expression is
4094/// invalid.
4095static Constant *
Nick Lewyckyc6501b12009-11-23 03:26:09 +00004096GetAddressedElementFromGlobal(GlobalVariable *GV,
Chris Lattner673e02b2004-10-12 01:49:27 +00004097 const std::vector<ConstantInt*> &Indices) {
4098 Constant *Init = GV->getInitializer();
4099 for (unsigned i = 0, e = Indices.size(); i != e; ++i) {
Reid Spencerb83eb642006-10-20 07:07:24 +00004100 uint64_t Idx = Indices[i]->getZExtValue();
Chris Lattner673e02b2004-10-12 01:49:27 +00004101 if (ConstantStruct *CS = dyn_cast<ConstantStruct>(Init)) {
4102 assert(Idx < CS->getNumOperands() && "Bad struct index!");
4103 Init = cast<Constant>(CS->getOperand(Idx));
4104 } else if (ConstantArray *CA = dyn_cast<ConstantArray>(Init)) {
4105 if (Idx >= CA->getNumOperands()) return 0; // Bogus program
4106 Init = cast<Constant>(CA->getOperand(Idx));
4107 } else if (isa<ConstantAggregateZero>(Init)) {
4108 if (const StructType *STy = dyn_cast<StructType>(Init->getType())) {
4109 assert(Idx < STy->getNumElements() && "Bad struct index!");
Owen Andersona7235ea2009-07-31 20:28:14 +00004110 Init = Constant::getNullValue(STy->getElementType(Idx));
Chris Lattner673e02b2004-10-12 01:49:27 +00004111 } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Init->getType())) {
4112 if (Idx >= ATy->getNumElements()) return 0; // Bogus program
Owen Andersona7235ea2009-07-31 20:28:14 +00004113 Init = Constant::getNullValue(ATy->getElementType());
Chris Lattner673e02b2004-10-12 01:49:27 +00004114 } else {
Torok Edwinc23197a2009-07-14 16:55:14 +00004115 llvm_unreachable("Unknown constant aggregate type!");
Chris Lattner673e02b2004-10-12 01:49:27 +00004116 }
4117 return 0;
4118 } else {
4119 return 0; // Unknown initializer type
4120 }
4121 }
4122 return Init;
4123}
4124
Dan Gohman46bdfb02009-02-24 18:55:53 +00004125/// ComputeLoadConstantCompareBackedgeTakenCount - Given an exit condition of
4126/// 'icmp op load X, cst', try to see if we can compute the backedge
4127/// execution count.
Dan Gohmanf6d009f2010-02-24 17:31:30 +00004128ScalarEvolution::BackedgeTakenInfo
Dan Gohman64a845e2009-06-24 04:48:43 +00004129ScalarEvolution::ComputeLoadConstantCompareBackedgeTakenCount(
4130 LoadInst *LI,
4131 Constant *RHS,
4132 const Loop *L,
4133 ICmpInst::Predicate predicate) {
Dan Gohman1c343752009-06-27 21:21:31 +00004134 if (LI->isVolatile()) return getCouldNotCompute();
Chris Lattner673e02b2004-10-12 01:49:27 +00004135
4136 // Check to see if the loaded pointer is a getelementptr of a global.
Dan Gohmanf6d009f2010-02-24 17:31:30 +00004137 // TODO: Use SCEV instead of manually grubbing with GEPs.
Chris Lattner673e02b2004-10-12 01:49:27 +00004138 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0));
Dan Gohman1c343752009-06-27 21:21:31 +00004139 if (!GEP) return getCouldNotCompute();
Chris Lattner673e02b2004-10-12 01:49:27 +00004140
4141 // Make sure that it is really a constant global we are gepping, with an
4142 // initializer, and make sure the first IDX is really 0.
4143 GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0));
Dan Gohman82555732009-08-19 18:20:44 +00004144 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
Chris Lattner673e02b2004-10-12 01:49:27 +00004145 GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) ||
4146 !cast<Constant>(GEP->getOperand(1))->isNullValue())
Dan Gohman1c343752009-06-27 21:21:31 +00004147 return getCouldNotCompute();
Chris Lattner673e02b2004-10-12 01:49:27 +00004148
4149 // Okay, we allow one non-constant index into the GEP instruction.
4150 Value *VarIdx = 0;
4151 std::vector<ConstantInt*> Indexes;
4152 unsigned VarIdxNum = 0;
4153 for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i)
4154 if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) {
4155 Indexes.push_back(CI);
4156 } else if (!isa<ConstantInt>(GEP->getOperand(i))) {
Dan Gohman1c343752009-06-27 21:21:31 +00004157 if (VarIdx) return getCouldNotCompute(); // Multiple non-constant idx's.
Chris Lattner673e02b2004-10-12 01:49:27 +00004158 VarIdx = GEP->getOperand(i);
4159 VarIdxNum = i-2;
4160 Indexes.push_back(0);
4161 }
4162
4163 // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant.
4164 // Check to see if X is a loop variant variable value now.
Dan Gohman0bba49c2009-07-07 17:06:11 +00004165 const SCEV *Idx = getSCEV(VarIdx);
Dan Gohmand594e6f2009-05-24 23:25:42 +00004166 Idx = getSCEVAtScope(Idx, L);
Chris Lattner673e02b2004-10-12 01:49:27 +00004167
4168 // We can only recognize very limited forms of loop index expressions, in
4169 // particular, only affine AddRec's like {C1,+,C2}.
Dan Gohman35738ac2009-05-04 22:30:44 +00004170 const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx);
Chris Lattner673e02b2004-10-12 01:49:27 +00004171 if (!IdxExpr || !IdxExpr->isAffine() || IdxExpr->isLoopInvariant(L) ||
4172 !isa<SCEVConstant>(IdxExpr->getOperand(0)) ||
4173 !isa<SCEVConstant>(IdxExpr->getOperand(1)))
Dan Gohman1c343752009-06-27 21:21:31 +00004174 return getCouldNotCompute();
Chris Lattner673e02b2004-10-12 01:49:27 +00004175
4176 unsigned MaxSteps = MaxBruteForceIterations;
4177 for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) {
Owen Andersoneed707b2009-07-24 23:12:02 +00004178 ConstantInt *ItCst = ConstantInt::get(
Owen Anderson9adc0ab2009-07-14 23:09:55 +00004179 cast<IntegerType>(IdxExpr->getType()), IterationNum);
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004180 ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this);
Chris Lattner673e02b2004-10-12 01:49:27 +00004181
4182 // Form the GEP offset.
4183 Indexes[VarIdxNum] = Val;
4184
Nick Lewyckyc6501b12009-11-23 03:26:09 +00004185 Constant *Result = GetAddressedElementFromGlobal(GV, Indexes);
Chris Lattner673e02b2004-10-12 01:49:27 +00004186 if (Result == 0) break; // Cannot compute!
4187
4188 // Evaluate the condition for this iteration.
Reid Spencere4d87aa2006-12-23 06:05:41 +00004189 Result = ConstantExpr::getICmp(predicate, Result, RHS);
Zhou Sheng6b6b6ef2007-01-11 12:24:14 +00004190 if (!isa<ConstantInt>(Result)) break; // Couldn't decide for sure
Reid Spencere8019bb2007-03-01 07:25:48 +00004191 if (cast<ConstantInt>(Result)->getValue().isMinValue()) {
Chris Lattner673e02b2004-10-12 01:49:27 +00004192#if 0
David Greene25e0e872009-12-23 22:18:14 +00004193 dbgs() << "\n***\n*** Computed loop count " << *ItCst
Dan Gohmanb7ef7292009-04-21 00:47:46 +00004194 << "\n*** From global " << *GV << "*** BB: " << *L->getHeader()
4195 << "***\n";
Chris Lattner673e02b2004-10-12 01:49:27 +00004196#endif
4197 ++NumArrayLenItCounts;
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004198 return getConstant(ItCst); // Found terminating iteration!
Chris Lattner673e02b2004-10-12 01:49:27 +00004199 }
4200 }
Dan Gohman1c343752009-06-27 21:21:31 +00004201 return getCouldNotCompute();
Chris Lattner673e02b2004-10-12 01:49:27 +00004202}
4203
4204
Chris Lattner3221ad02004-04-17 22:58:41 +00004205/// CanConstantFold - Return true if we can constant fold an instruction of the
4206/// specified type, assuming that all operands were constants.
4207static bool CanConstantFold(const Instruction *I) {
Reid Spencer832254e2007-02-02 02:16:23 +00004208 if (isa<BinaryOperator>(I) || isa<CmpInst>(I) ||
Chris Lattner3221ad02004-04-17 22:58:41 +00004209 isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I))
4210 return true;
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004211
Chris Lattner3221ad02004-04-17 22:58:41 +00004212 if (const CallInst *CI = dyn_cast<CallInst>(I))
4213 if (const Function *F = CI->getCalledFunction())
Dan Gohmanfa9b80e2008-01-31 01:05:10 +00004214 return canConstantFoldCallTo(F);
Chris Lattner3221ad02004-04-17 22:58:41 +00004215 return false;
Chris Lattner7980fb92004-04-17 18:36:24 +00004216}
4217
Chris Lattner3221ad02004-04-17 22:58:41 +00004218/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
4219/// in the loop that V is derived from. We allow arbitrary operations along the
4220/// way, but the operands of an operation must either be constants or a value
4221/// derived from a constant PHI. If this expression does not fit with these
4222/// constraints, return null.
4223static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
4224 // If this is not an instruction, or if this is an instruction outside of the
4225 // loop, it can't be derived from a loop PHI.
4226 Instruction *I = dyn_cast<Instruction>(V);
Dan Gohman92329c72009-12-18 01:24:09 +00004227 if (I == 0 || !L->contains(I)) return 0;
Chris Lattner3221ad02004-04-17 22:58:41 +00004228
Anton Korobeynikovae9f3a32008-02-20 11:08:44 +00004229 if (PHINode *PN = dyn_cast<PHINode>(I)) {
Chris Lattner3221ad02004-04-17 22:58:41 +00004230 if (L->getHeader() == I->getParent())
4231 return PN;
4232 else
4233 // We don't currently keep track of the control flow needed to evaluate
4234 // PHIs, so we cannot handle PHIs inside of loops.
4235 return 0;
Anton Korobeynikovae9f3a32008-02-20 11:08:44 +00004236 }
Chris Lattner3221ad02004-04-17 22:58:41 +00004237
4238 // If we won't be able to constant fold this expression even if the operands
4239 // are constants, return early.
4240 if (!CanConstantFold(I)) return 0;
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004241
Chris Lattner3221ad02004-04-17 22:58:41 +00004242 // Otherwise, we can evaluate this instruction if all of its operands are
4243 // constant or derived from a PHI node themselves.
4244 PHINode *PHI = 0;
4245 for (unsigned Op = 0, e = I->getNumOperands(); Op != e; ++Op)
Dan Gohman9d4588f2010-06-22 13:15:46 +00004246 if (!isa<Constant>(I->getOperand(Op))) {
Chris Lattner3221ad02004-04-17 22:58:41 +00004247 PHINode *P = getConstantEvolvingPHI(I->getOperand(Op), L);
4248 if (P == 0) return 0; // Not evolving from PHI
4249 if (PHI == 0)
4250 PHI = P;
4251 else if (PHI != P)
4252 return 0; // Evolving from multiple different PHIs.
4253 }
4254
4255 // This is a expression evolving from a constant PHI!
4256 return PHI;
4257}
4258
4259/// EvaluateExpression - Given an expression that passes the
4260/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
4261/// in the loop has the value PHIVal. If we can't fold this expression for some
4262/// reason, return null.
Dan Gohman1ba3b6c2009-11-09 23:34:17 +00004263static Constant *EvaluateExpression(Value *V, Constant *PHIVal,
4264 const TargetData *TD) {
Chris Lattner3221ad02004-04-17 22:58:41 +00004265 if (isa<PHINode>(V)) return PHIVal;
Reid Spencere8404342004-07-18 00:18:30 +00004266 if (Constant *C = dyn_cast<Constant>(V)) return C;
Chris Lattner3221ad02004-04-17 22:58:41 +00004267 Instruction *I = cast<Instruction>(V);
4268
Dan Gohman9d4588f2010-06-22 13:15:46 +00004269 std::vector<Constant*> Operands(I->getNumOperands());
Chris Lattner3221ad02004-04-17 22:58:41 +00004270
4271 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
Dan Gohman1ba3b6c2009-11-09 23:34:17 +00004272 Operands[i] = EvaluateExpression(I->getOperand(i), PHIVal, TD);
Chris Lattner3221ad02004-04-17 22:58:41 +00004273 if (Operands[i] == 0) return 0;
4274 }
4275
Chris Lattnerf286f6f2007-12-10 22:53:04 +00004276 if (const CmpInst *CI = dyn_cast<CmpInst>(I))
Chris Lattner8f73dea2009-11-09 23:06:58 +00004277 return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
Dan Gohman1ba3b6c2009-11-09 23:34:17 +00004278 Operands[1], TD);
Chris Lattner8f73dea2009-11-09 23:06:58 +00004279 return ConstantFoldInstOperands(I->getOpcode(), I->getType(),
Dan Gohman1ba3b6c2009-11-09 23:34:17 +00004280 &Operands[0], Operands.size(), TD);
Chris Lattner3221ad02004-04-17 22:58:41 +00004281}
4282
4283/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
4284/// in the header of its containing loop, we know the loop executes a
4285/// constant number of times, and the PHI node is just a recurrence
4286/// involving constants, fold it.
Dan Gohman64a845e2009-06-24 04:48:43 +00004287Constant *
4288ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
Dan Gohman5d984912009-12-18 01:14:11 +00004289 const APInt &BEs,
Dan Gohman64a845e2009-06-24 04:48:43 +00004290 const Loop *L) {
Dan Gohman8d9c7a62010-08-16 16:30:01 +00004291 std::map<PHINode*, Constant*>::const_iterator I =
Chris Lattner3221ad02004-04-17 22:58:41 +00004292 ConstantEvolutionLoopExitValue.find(PN);
4293 if (I != ConstantEvolutionLoopExitValue.end())
4294 return I->second;
4295
Dan Gohmane0567812010-04-08 23:03:40 +00004296 if (BEs.ugt(MaxBruteForceIterations))
Chris Lattner3221ad02004-04-17 22:58:41 +00004297 return ConstantEvolutionLoopExitValue[PN] = 0; // Not going to evaluate it.
4298
4299 Constant *&RetVal = ConstantEvolutionLoopExitValue[PN];
4300
4301 // Since the loop is canonicalized, the PHI node must have two entries. One
4302 // entry must be a constant (coming in from outside of the loop), and the
4303 // second must be derived from the same PHI.
4304 bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
4305 Constant *StartCST =
4306 dyn_cast<Constant>(PN->getIncomingValue(!SecondIsBackedge));
4307 if (StartCST == 0)
4308 return RetVal = 0; // Must be a constant.
4309
4310 Value *BEValue = PN->getIncomingValue(SecondIsBackedge);
Dan Gohman9d4588f2010-06-22 13:15:46 +00004311 if (getConstantEvolvingPHI(BEValue, L) != PN &&
4312 !isa<Constant>(BEValue))
Chris Lattner3221ad02004-04-17 22:58:41 +00004313 return RetVal = 0; // Not derived from same PHI.
4314
4315 // Execute the loop symbolically to determine the exit value.
Dan Gohman46bdfb02009-02-24 18:55:53 +00004316 if (BEs.getActiveBits() >= 32)
Reid Spencere8019bb2007-03-01 07:25:48 +00004317 return RetVal = 0; // More than 2^32-1 iterations?? Not doing it!
Chris Lattner3221ad02004-04-17 22:58:41 +00004318
Dan Gohman46bdfb02009-02-24 18:55:53 +00004319 unsigned NumIterations = BEs.getZExtValue(); // must be in range
Reid Spencere8019bb2007-03-01 07:25:48 +00004320 unsigned IterationNum = 0;
Chris Lattner3221ad02004-04-17 22:58:41 +00004321 for (Constant *PHIVal = StartCST; ; ++IterationNum) {
4322 if (IterationNum == NumIterations)
4323 return RetVal = PHIVal; // Got exit value!
4324
4325 // Compute the value of the PHI node for the next iteration.
Dan Gohman1ba3b6c2009-11-09 23:34:17 +00004326 Constant *NextPHI = EvaluateExpression(BEValue, PHIVal, TD);
Chris Lattner3221ad02004-04-17 22:58:41 +00004327 if (NextPHI == PHIVal)
4328 return RetVal = NextPHI; // Stopped evolving!
4329 if (NextPHI == 0)
4330 return 0; // Couldn't evaluate!
4331 PHIVal = NextPHI;
4332 }
4333}
4334
Dan Gohman07ad19b2009-07-27 16:09:48 +00004335/// ComputeBackedgeTakenCountExhaustively - If the loop is known to execute a
Chris Lattner7980fb92004-04-17 18:36:24 +00004336/// constant number of times (the condition evolves only from constants),
4337/// try to evaluate a few iterations of the loop until we get the exit
4338/// condition gets a value of ExitWhen (true or false). If we cannot
Dan Gohman1c343752009-06-27 21:21:31 +00004339/// evaluate the trip count of the loop, return getCouldNotCompute().
Dan Gohman64a845e2009-06-24 04:48:43 +00004340const SCEV *
4341ScalarEvolution::ComputeBackedgeTakenCountExhaustively(const Loop *L,
4342 Value *Cond,
4343 bool ExitWhen) {
Chris Lattner7980fb92004-04-17 18:36:24 +00004344 PHINode *PN = getConstantEvolvingPHI(Cond, L);
Dan Gohman1c343752009-06-27 21:21:31 +00004345 if (PN == 0) return getCouldNotCompute();
Chris Lattner7980fb92004-04-17 18:36:24 +00004346
Dan Gohmanb92654d2010-06-19 14:17:24 +00004347 // If the loop is canonicalized, the PHI will have exactly two entries.
4348 // That's the only form we support here.
4349 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
4350
4351 // One entry must be a constant (coming in from outside of the loop), and the
Chris Lattner7980fb92004-04-17 18:36:24 +00004352 // second must be derived from the same PHI.
4353 bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
4354 Constant *StartCST =
4355 dyn_cast<Constant>(PN->getIncomingValue(!SecondIsBackedge));
Dan Gohman1c343752009-06-27 21:21:31 +00004356 if (StartCST == 0) return getCouldNotCompute(); // Must be a constant.
Chris Lattner7980fb92004-04-17 18:36:24 +00004357
4358 Value *BEValue = PN->getIncomingValue(SecondIsBackedge);
Dan Gohman9d4588f2010-06-22 13:15:46 +00004359 if (getConstantEvolvingPHI(BEValue, L) != PN &&
4360 !isa<Constant>(BEValue))
4361 return getCouldNotCompute(); // Not derived from same PHI.
Chris Lattner7980fb92004-04-17 18:36:24 +00004362
4363 // Okay, we find a PHI node that defines the trip count of this loop. Execute
4364 // the loop symbolically to determine when the condition gets a value of
4365 // "ExitWhen".
4366 unsigned IterationNum = 0;
4367 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
4368 for (Constant *PHIVal = StartCST;
4369 IterationNum != MaxIterations; ++IterationNum) {
Zhou Sheng6b6b6ef2007-01-11 12:24:14 +00004370 ConstantInt *CondVal =
Dan Gohman1ba3b6c2009-11-09 23:34:17 +00004371 dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, PHIVal, TD));
Chris Lattner3221ad02004-04-17 22:58:41 +00004372
Zhou Sheng6b6b6ef2007-01-11 12:24:14 +00004373 // Couldn't symbolically evaluate.
Dan Gohman1c343752009-06-27 21:21:31 +00004374 if (!CondVal) return getCouldNotCompute();
Zhou Sheng6b6b6ef2007-01-11 12:24:14 +00004375
Reid Spencere8019bb2007-03-01 07:25:48 +00004376 if (CondVal->getValue() == uint64_t(ExitWhen)) {
Chris Lattner7980fb92004-04-17 18:36:24 +00004377 ++NumBruteForceTripCountsComputed;
Owen Anderson1d0be152009-08-13 21:58:54 +00004378 return getConstant(Type::getInt32Ty(getContext()), IterationNum);
Chris Lattner7980fb92004-04-17 18:36:24 +00004379 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004380
Chris Lattner3221ad02004-04-17 22:58:41 +00004381 // Compute the value of the PHI node for the next iteration.
Dan Gohman1ba3b6c2009-11-09 23:34:17 +00004382 Constant *NextPHI = EvaluateExpression(BEValue, PHIVal, TD);
Chris Lattner3221ad02004-04-17 22:58:41 +00004383 if (NextPHI == 0 || NextPHI == PHIVal)
Dan Gohman1c343752009-06-27 21:21:31 +00004384 return getCouldNotCompute();// Couldn't evaluate or not making progress...
Chris Lattner3221ad02004-04-17 22:58:41 +00004385 PHIVal = NextPHI;
Chris Lattner7980fb92004-04-17 18:36:24 +00004386 }
4387
4388 // Too many iterations were needed to evaluate.
Dan Gohman1c343752009-06-27 21:21:31 +00004389 return getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00004390}
4391
Dan Gohmane7125f42009-09-03 15:00:26 +00004392/// getSCEVAtScope - Return a SCEV expression for the specified value
Dan Gohman66a7e852009-05-08 20:38:54 +00004393/// at the specified scope in the program. The L value specifies a loop
4394/// nest to evaluate the expression at, where null is the top-level or a
4395/// specified loop is immediately inside of the loop.
4396///
4397/// This method can be used to compute the exit value for a variable defined
4398/// in a loop by querying what the value will hold in the parent loop.
4399///
Dan Gohmand594e6f2009-05-24 23:25:42 +00004400/// In the case that a relevant loop exit value cannot be computed, the
4401/// original value V is returned.
Dan Gohman0bba49c2009-07-07 17:06:11 +00004402const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
Dan Gohman42214892009-08-31 21:15:23 +00004403 // Check to see if we've folded this expression at this loop before.
4404 std::map<const Loop *, const SCEV *> &Values = ValuesAtScopes[V];
4405 std::pair<std::map<const Loop *, const SCEV *>::iterator, bool> Pair =
4406 Values.insert(std::make_pair(L, static_cast<const SCEV *>(0)));
4407 if (!Pair.second)
4408 return Pair.first->second ? Pair.first->second : V;
Chris Lattner53e677a2004-04-02 20:23:17 +00004409
Dan Gohman42214892009-08-31 21:15:23 +00004410 // Otherwise compute it.
4411 const SCEV *C = computeSCEVAtScope(V, L);
Dan Gohmana5505cb2009-08-31 21:58:28 +00004412 ValuesAtScopes[V][L] = C;
Dan Gohman42214892009-08-31 21:15:23 +00004413 return C;
4414}
4415
4416const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
Chris Lattner3221ad02004-04-17 22:58:41 +00004417 if (isa<SCEVConstant>(V)) return V;
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004418
Nick Lewycky3e630762008-02-20 06:48:22 +00004419 // If this instruction is evolved from a constant-evolving PHI, compute the
Chris Lattner3221ad02004-04-17 22:58:41 +00004420 // exit value from the loop without using SCEVs.
Dan Gohman622ed672009-05-04 22:02:23 +00004421 if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) {
Chris Lattner3221ad02004-04-17 22:58:41 +00004422 if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) {
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004423 const Loop *LI = (*this->LI)[I->getParent()];
Chris Lattner3221ad02004-04-17 22:58:41 +00004424 if (LI && LI->getParentLoop() == L) // Looking for loop exit value.
4425 if (PHINode *PN = dyn_cast<PHINode>(I))
4426 if (PN->getParent() == LI->getHeader()) {
4427 // Okay, there is no closed form solution for the PHI node. Check
Dan Gohman46bdfb02009-02-24 18:55:53 +00004428 // to see if the loop that contains it has a known backedge-taken
4429 // count. If so, we may be able to force computation of the exit
4430 // value.
Dan Gohman0bba49c2009-07-07 17:06:11 +00004431 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI);
Dan Gohman622ed672009-05-04 22:02:23 +00004432 if (const SCEVConstant *BTCC =
Dan Gohman46bdfb02009-02-24 18:55:53 +00004433 dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
Chris Lattner3221ad02004-04-17 22:58:41 +00004434 // Okay, we know how many times the containing loop executes. If
4435 // this is a constant evolving PHI node, get the final value at
4436 // the specified iteration number.
4437 Constant *RV = getConstantEvolutionLoopExitValue(PN,
Dan Gohman46bdfb02009-02-24 18:55:53 +00004438 BTCC->getValue()->getValue(),
Chris Lattner3221ad02004-04-17 22:58:41 +00004439 LI);
Dan Gohman09987962009-06-29 21:31:18 +00004440 if (RV) return getSCEV(RV);
Chris Lattner3221ad02004-04-17 22:58:41 +00004441 }
4442 }
4443
Reid Spencer09906f32006-12-04 21:33:23 +00004444 // Okay, this is an expression that we cannot symbolically evaluate
Chris Lattner3221ad02004-04-17 22:58:41 +00004445 // into a SCEV. Check to see if it's possible to symbolically evaluate
Reid Spencer09906f32006-12-04 21:33:23 +00004446 // the arguments into constants, and if so, try to constant propagate the
Chris Lattner3221ad02004-04-17 22:58:41 +00004447 // result. This is particularly useful for computing loop exit values.
4448 if (CanConstantFold(I)) {
Dan Gohman11046452010-06-29 23:43:06 +00004449 SmallVector<Constant *, 4> Operands;
4450 bool MadeImprovement = false;
Chris Lattner3221ad02004-04-17 22:58:41 +00004451 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
4452 Value *Op = I->getOperand(i);
4453 if (Constant *C = dyn_cast<Constant>(Op)) {
4454 Operands.push_back(C);
Dan Gohman11046452010-06-29 23:43:06 +00004455 continue;
Chris Lattner3221ad02004-04-17 22:58:41 +00004456 }
Dan Gohman11046452010-06-29 23:43:06 +00004457
4458 // If any of the operands is non-constant and if they are
4459 // non-integer and non-pointer, don't even try to analyze them
4460 // with scev techniques.
4461 if (!isSCEVable(Op->getType()))
4462 return V;
4463
4464 const SCEV *OrigV = getSCEV(Op);
4465 const SCEV *OpV = getSCEVAtScope(OrigV, L);
4466 MadeImprovement |= OrigV != OpV;
4467
4468 Constant *C = 0;
4469 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(OpV))
4470 C = SC->getValue();
4471 if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(OpV))
4472 C = dyn_cast<Constant>(SU->getValue());
4473 if (!C) return V;
4474 if (C->getType() != Op->getType())
4475 C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
4476 Op->getType(),
4477 false),
4478 C, Op->getType());
4479 Operands.push_back(C);
Chris Lattner3221ad02004-04-17 22:58:41 +00004480 }
Dan Gohman64a845e2009-06-24 04:48:43 +00004481
Dan Gohman11046452010-06-29 23:43:06 +00004482 // Check to see if getSCEVAtScope actually made an improvement.
4483 if (MadeImprovement) {
4484 Constant *C = 0;
4485 if (const CmpInst *CI = dyn_cast<CmpInst>(I))
4486 C = ConstantFoldCompareInstOperands(CI->getPredicate(),
4487 Operands[0], Operands[1], TD);
4488 else
4489 C = ConstantFoldInstOperands(I->getOpcode(), I->getType(),
4490 &Operands[0], Operands.size(), TD);
4491 if (!C) return V;
Dan Gohmane177c9a2010-02-24 19:31:47 +00004492 return getSCEV(C);
Dan Gohman11046452010-06-29 23:43:06 +00004493 }
Chris Lattner3221ad02004-04-17 22:58:41 +00004494 }
4495 }
4496
4497 // This is some other type of SCEVUnknown, just return it.
4498 return V;
4499 }
4500
Dan Gohman622ed672009-05-04 22:02:23 +00004501 if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004502 // Avoid performing the look-up in the common case where the specified
4503 // expression has no loop-variant portions.
4504 for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00004505 const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
Chris Lattner53e677a2004-04-02 20:23:17 +00004506 if (OpAtScope != Comm->getOperand(i)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004507 // Okay, at least one of these operands is loop variant but might be
4508 // foldable. Build a new instance of the folded commutative expression.
Dan Gohman64a845e2009-06-24 04:48:43 +00004509 SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(),
4510 Comm->op_begin()+i);
Chris Lattner53e677a2004-04-02 20:23:17 +00004511 NewOps.push_back(OpAtScope);
4512
4513 for (++i; i != e; ++i) {
4514 OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
Chris Lattner53e677a2004-04-02 20:23:17 +00004515 NewOps.push_back(OpAtScope);
4516 }
4517 if (isa<SCEVAddExpr>(Comm))
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004518 return getAddExpr(NewOps);
Nick Lewyckyc54c5612007-11-25 22:41:31 +00004519 if (isa<SCEVMulExpr>(Comm))
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004520 return getMulExpr(NewOps);
Nick Lewyckyc54c5612007-11-25 22:41:31 +00004521 if (isa<SCEVSMaxExpr>(Comm))
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004522 return getSMaxExpr(NewOps);
Nick Lewycky3e630762008-02-20 06:48:22 +00004523 if (isa<SCEVUMaxExpr>(Comm))
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004524 return getUMaxExpr(NewOps);
Torok Edwinc23197a2009-07-14 16:55:14 +00004525 llvm_unreachable("Unknown commutative SCEV type!");
Chris Lattner53e677a2004-04-02 20:23:17 +00004526 }
4527 }
4528 // If we got here, all operands are loop invariant.
4529 return Comm;
4530 }
4531
Dan Gohman622ed672009-05-04 22:02:23 +00004532 if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00004533 const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L);
4534 const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L);
Nick Lewycky789558d2009-01-13 09:18:58 +00004535 if (LHS == Div->getLHS() && RHS == Div->getRHS())
4536 return Div; // must be loop invariant
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004537 return getUDivExpr(LHS, RHS);
Chris Lattner53e677a2004-04-02 20:23:17 +00004538 }
4539
4540 // If this is a loop recurrence for a loop that does not contain L, then we
4541 // are dealing with the final value computed by the loop.
Dan Gohman622ed672009-05-04 22:02:23 +00004542 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
Dan Gohman11046452010-06-29 23:43:06 +00004543 // First, attempt to evaluate each operand.
4544 // Avoid performing the look-up in the common case where the specified
4545 // expression has no loop-variant portions.
4546 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
4547 const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L);
4548 if (OpAtScope == AddRec->getOperand(i))
4549 continue;
4550
4551 // Okay, at least one of these operands is loop variant but might be
4552 // foldable. Build a new instance of the folded commutative expression.
4553 SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(),
4554 AddRec->op_begin()+i);
4555 NewOps.push_back(OpAtScope);
4556 for (++i; i != e; ++i)
4557 NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L));
4558
4559 AddRec = cast<SCEVAddRecExpr>(getAddRecExpr(NewOps, AddRec->getLoop()));
4560 break;
4561 }
4562
4563 // If the scope is outside the addrec's loop, evaluate it by using the
4564 // loop exit value of the addrec.
4565 if (!AddRec->getLoop()->contains(L)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004566 // To evaluate this recurrence, we need to know how many times the AddRec
4567 // loop iterates. Compute this now.
Dan Gohman0bba49c2009-07-07 17:06:11 +00004568 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop());
Dan Gohman1c343752009-06-27 21:21:31 +00004569 if (BackedgeTakenCount == getCouldNotCompute()) return AddRec;
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004570
Eli Friedmanb42a6262008-08-04 23:49:06 +00004571 // Then, evaluate the AddRec.
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004572 return AddRec->evaluateAtIteration(BackedgeTakenCount, *this);
Chris Lattner53e677a2004-04-02 20:23:17 +00004573 }
Dan Gohman11046452010-06-29 23:43:06 +00004574
Dan Gohmand594e6f2009-05-24 23:25:42 +00004575 return AddRec;
Chris Lattner53e677a2004-04-02 20:23:17 +00004576 }
4577
Dan Gohman622ed672009-05-04 22:02:23 +00004578 if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00004579 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
Dan Gohmaneb3948b2009-04-29 22:29:01 +00004580 if (Op == Cast->getOperand())
4581 return Cast; // must be loop invariant
4582 return getZeroExtendExpr(Op, Cast->getType());
4583 }
4584
Dan Gohman622ed672009-05-04 22:02:23 +00004585 if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00004586 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
Dan Gohmaneb3948b2009-04-29 22:29:01 +00004587 if (Op == Cast->getOperand())
4588 return Cast; // must be loop invariant
4589 return getSignExtendExpr(Op, Cast->getType());
4590 }
4591
Dan Gohman622ed672009-05-04 22:02:23 +00004592 if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00004593 const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
Dan Gohmaneb3948b2009-04-29 22:29:01 +00004594 if (Op == Cast->getOperand())
4595 return Cast; // must be loop invariant
4596 return getTruncateExpr(Op, Cast->getType());
4597 }
4598
Torok Edwinc23197a2009-07-14 16:55:14 +00004599 llvm_unreachable("Unknown SCEV type!");
Daniel Dunbar8c562e22009-05-18 16:43:04 +00004600 return 0;
Chris Lattner53e677a2004-04-02 20:23:17 +00004601}
4602
Dan Gohman66a7e852009-05-08 20:38:54 +00004603/// getSCEVAtScope - This is a convenience function which does
4604/// getSCEVAtScope(getSCEV(V), L).
Dan Gohman0bba49c2009-07-07 17:06:11 +00004605const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004606 return getSCEVAtScope(getSCEV(V), L);
4607}
4608
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004609/// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the
4610/// following equation:
4611///
4612/// A * X = B (mod N)
4613///
4614/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
4615/// A and B isn't important.
4616///
4617/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
Dan Gohman0bba49c2009-07-07 17:06:11 +00004618static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B,
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004619 ScalarEvolution &SE) {
4620 uint32_t BW = A.getBitWidth();
4621 assert(BW == B.getBitWidth() && "Bit widths must be the same.");
4622 assert(A != 0 && "A must be non-zero.");
4623
4624 // 1. D = gcd(A, N)
4625 //
4626 // The gcd of A and N may have only one prime factor: 2. The number of
4627 // trailing zeros in A is its multiplicity
4628 uint32_t Mult2 = A.countTrailingZeros();
4629 // D = 2^Mult2
4630
4631 // 2. Check if B is divisible by D.
4632 //
4633 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
4634 // is not less than multiplicity of this prime factor for D.
4635 if (B.countTrailingZeros() < Mult2)
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00004636 return SE.getCouldNotCompute();
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004637
4638 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
4639 // modulo (N / D).
4640 //
4641 // (N / D) may need BW+1 bits in its representation. Hence, we'll use this
4642 // bit width during computations.
4643 APInt AD = A.lshr(Mult2).zext(BW + 1); // AD = A / D
4644 APInt Mod(BW + 1, 0);
4645 Mod.set(BW - Mult2); // Mod = N / D
4646 APInt I = AD.multiplicativeInverse(Mod);
4647
4648 // 4. Compute the minimum unsigned root of the equation:
4649 // I * (B / D) mod (N / D)
4650 APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod);
4651
4652 // The result is guaranteed to be less than 2^BW so we may truncate it to BW
4653 // bits.
4654 return SE.getConstant(Result.trunc(BW));
4655}
Chris Lattner53e677a2004-04-02 20:23:17 +00004656
4657/// SolveQuadraticEquation - Find the roots of the quadratic equation for the
4658/// given quadratic chrec {L,+,M,+,N}. This returns either the two roots (which
4659/// might be the same) or two SCEVCouldNotCompute objects.
4660///
Dan Gohman0bba49c2009-07-07 17:06:11 +00004661static std::pair<const SCEV *,const SCEV *>
Dan Gohman246b2562007-10-22 18:31:58 +00004662SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004663 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
Dan Gohman35738ac2009-05-04 22:30:44 +00004664 const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0));
4665 const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1));
4666 const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2));
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004667
Chris Lattner53e677a2004-04-02 20:23:17 +00004668 // We currently can only solve this if the coefficients are constants.
Reid Spencere8019bb2007-03-01 07:25:48 +00004669 if (!LC || !MC || !NC) {
Dan Gohman35738ac2009-05-04 22:30:44 +00004670 const SCEV *CNC = SE.getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00004671 return std::make_pair(CNC, CNC);
4672 }
4673
Reid Spencere8019bb2007-03-01 07:25:48 +00004674 uint32_t BitWidth = LC->getValue()->getValue().getBitWidth();
Chris Lattnerfe560b82007-04-15 19:52:49 +00004675 const APInt &L = LC->getValue()->getValue();
4676 const APInt &M = MC->getValue()->getValue();
4677 const APInt &N = NC->getValue()->getValue();
Reid Spencere8019bb2007-03-01 07:25:48 +00004678 APInt Two(BitWidth, 2);
4679 APInt Four(BitWidth, 4);
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004680
Dan Gohman64a845e2009-06-24 04:48:43 +00004681 {
Reid Spencere8019bb2007-03-01 07:25:48 +00004682 using namespace APIntOps;
Zhou Sheng414de4d2007-04-07 17:48:27 +00004683 const APInt& C = L;
Reid Spencere8019bb2007-03-01 07:25:48 +00004684 // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C
4685 // The B coefficient is M-N/2
4686 APInt B(M);
4687 B -= sdiv(N,Two);
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004688
Reid Spencere8019bb2007-03-01 07:25:48 +00004689 // The A coefficient is N/2
Zhou Sheng414de4d2007-04-07 17:48:27 +00004690 APInt A(N.sdiv(Two));
Chris Lattner53e677a2004-04-02 20:23:17 +00004691
Reid Spencere8019bb2007-03-01 07:25:48 +00004692 // Compute the B^2-4ac term.
4693 APInt SqrtTerm(B);
4694 SqrtTerm *= B;
4695 SqrtTerm -= Four * (A * C);
Chris Lattner53e677a2004-04-02 20:23:17 +00004696
Reid Spencere8019bb2007-03-01 07:25:48 +00004697 // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest
4698 // integer value or else APInt::sqrt() will assert.
4699 APInt SqrtVal(SqrtTerm.sqrt());
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004700
Dan Gohman64a845e2009-06-24 04:48:43 +00004701 // Compute the two solutions for the quadratic formula.
Reid Spencere8019bb2007-03-01 07:25:48 +00004702 // The divisions must be performed as signed divisions.
4703 APInt NegB(-B);
Reid Spencer3e35c8d2007-04-16 02:24:41 +00004704 APInt TwoA( A << 1 );
Nick Lewycky8f4d5eb2008-11-03 02:43:49 +00004705 if (TwoA.isMinValue()) {
Dan Gohman35738ac2009-05-04 22:30:44 +00004706 const SCEV *CNC = SE.getCouldNotCompute();
Nick Lewycky8f4d5eb2008-11-03 02:43:49 +00004707 return std::make_pair(CNC, CNC);
4708 }
4709
Owen Andersone922c022009-07-22 00:24:57 +00004710 LLVMContext &Context = SE.getContext();
Owen Anderson76f600b2009-07-06 22:37:39 +00004711
4712 ConstantInt *Solution1 =
Owen Andersoneed707b2009-07-24 23:12:02 +00004713 ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA));
Owen Anderson76f600b2009-07-06 22:37:39 +00004714 ConstantInt *Solution2 =
Owen Andersoneed707b2009-07-24 23:12:02 +00004715 ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA));
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004716
Dan Gohman64a845e2009-06-24 04:48:43 +00004717 return std::make_pair(SE.getConstant(Solution1),
Dan Gohman246b2562007-10-22 18:31:58 +00004718 SE.getConstant(Solution2));
Reid Spencere8019bb2007-03-01 07:25:48 +00004719 } // end APIntOps namespace
Chris Lattner53e677a2004-04-02 20:23:17 +00004720}
4721
4722/// HowFarToZero - Return the number of times a backedge comparing the specified
Dan Gohman86fbf2f2009-06-06 14:37:11 +00004723/// value to zero will execute. If not computable, return CouldNotCompute.
Dan Gohmanf6d009f2010-02-24 17:31:30 +00004724ScalarEvolution::BackedgeTakenInfo
4725ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004726 // If the value is a constant
Dan Gohman622ed672009-05-04 22:02:23 +00004727 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004728 // If the value is already zero, the branch will execute zero times.
Reid Spencercae57542007-03-02 00:28:52 +00004729 if (C->getValue()->isZero()) return C;
Dan Gohman1c343752009-06-27 21:21:31 +00004730 return getCouldNotCompute(); // Otherwise it will loop infinitely.
Chris Lattner53e677a2004-04-02 20:23:17 +00004731 }
4732
Dan Gohman35738ac2009-05-04 22:30:44 +00004733 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V);
Chris Lattner53e677a2004-04-02 20:23:17 +00004734 if (!AddRec || AddRec->getLoop() != L)
Dan Gohman1c343752009-06-27 21:21:31 +00004735 return getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00004736
4737 if (AddRec->isAffine()) {
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004738 // If this is an affine expression, the execution count of this branch is
4739 // the minimum unsigned root of the following equation:
Chris Lattner53e677a2004-04-02 20:23:17 +00004740 //
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004741 // Start + Step*N = 0 (mod 2^BW)
Chris Lattner53e677a2004-04-02 20:23:17 +00004742 //
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004743 // equivalent to:
4744 //
4745 // Step*N = -Start (mod 2^BW)
4746 //
4747 // where BW is the common bit width of Start and Step.
4748
Chris Lattner53e677a2004-04-02 20:23:17 +00004749 // Get the initial value for the loop.
Dan Gohman64a845e2009-06-24 04:48:43 +00004750 const SCEV *Start = getSCEVAtScope(AddRec->getStart(),
4751 L->getParentLoop());
4752 const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1),
4753 L->getParentLoop());
Chris Lattner53e677a2004-04-02 20:23:17 +00004754
Dan Gohman622ed672009-05-04 22:02:23 +00004755 if (const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step)) {
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004756 // For now we handle only constant steps.
Chris Lattner53e677a2004-04-02 20:23:17 +00004757
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004758 // First, handle unitary steps.
4759 if (StepC->getValue()->equalsInt(1)) // 1*N = -Start (mod 2^BW), so:
Dan Gohman4c0d5d52009-08-20 16:42:55 +00004760 return getNegativeSCEV(Start); // N = -Start (as unsigned)
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004761 if (StepC->getValue()->isAllOnesValue()) // -1*N = -Start (mod 2^BW), so:
4762 return Start; // N = Start (as unsigned)
4763
4764 // Then, try to solve the above equation provided that Start is constant.
Dan Gohman622ed672009-05-04 22:02:23 +00004765 if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start))
Wojciech Matyjewiczde0f2382008-07-20 15:55:14 +00004766 return SolveLinEquationWithOverflow(StepC->getValue()->getValue(),
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004767 -StartC->getValue()->getValue(),
4768 *this);
Chris Lattner53e677a2004-04-02 20:23:17 +00004769 }
Duncan Sandsb0bc6c32010-02-15 16:12:20 +00004770 } else if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004771 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
4772 // the quadratic equation to solve it.
Dan Gohman0bba49c2009-07-07 17:06:11 +00004773 std::pair<const SCEV *,const SCEV *> Roots = SolveQuadraticEquation(AddRec,
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004774 *this);
Dan Gohman35738ac2009-05-04 22:30:44 +00004775 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first);
4776 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second);
Chris Lattner53e677a2004-04-02 20:23:17 +00004777 if (R1) {
Chris Lattnerd18d9dc2004-04-02 20:26:46 +00004778#if 0
David Greene25e0e872009-12-23 22:18:14 +00004779 dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1
Dan Gohmanb7ef7292009-04-21 00:47:46 +00004780 << " sol#2: " << *R2 << "\n";
Chris Lattnerd18d9dc2004-04-02 20:26:46 +00004781#endif
Chris Lattner53e677a2004-04-02 20:23:17 +00004782 // Pick the smallest positive root value.
Zhou Sheng6b6b6ef2007-01-11 12:24:14 +00004783 if (ConstantInt *CB =
Owen Andersonbaf3c402009-07-29 18:55:55 +00004784 dyn_cast<ConstantInt>(ConstantExpr::getICmp(ICmpInst::ICMP_ULT,
Reid Spencere4d87aa2006-12-23 06:05:41 +00004785 R1->getValue(), R2->getValue()))) {
Reid Spencer579dca12007-01-12 04:24:46 +00004786 if (CB->getZExtValue() == false)
Chris Lattner53e677a2004-04-02 20:23:17 +00004787 std::swap(R1, R2); // R1 is the minimum root now.
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004788
Chris Lattner53e677a2004-04-02 20:23:17 +00004789 // We can only use this value if the chrec ends up with an exact zero
4790 // value at this index. When solving for "X*X != 5", for example, we
4791 // should not accept a root of 2.
Dan Gohman0bba49c2009-07-07 17:06:11 +00004792 const SCEV *Val = AddRec->evaluateAtIteration(R1, *this);
Dan Gohmancfeb6a42008-06-18 16:23:07 +00004793 if (Val->isZero())
4794 return R1; // We found a quadratic root!
Chris Lattner53e677a2004-04-02 20:23:17 +00004795 }
4796 }
4797 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004798
Dan Gohman1c343752009-06-27 21:21:31 +00004799 return getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00004800}
4801
4802/// HowFarToNonZero - Return the number of times a backedge checking the
4803/// specified value for nonzero will execute. If not computable, return
Dan Gohman86fbf2f2009-06-06 14:37:11 +00004804/// CouldNotCompute
Dan Gohmanf6d009f2010-02-24 17:31:30 +00004805ScalarEvolution::BackedgeTakenInfo
4806ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) {
Chris Lattner53e677a2004-04-02 20:23:17 +00004807 // Loops that look like: while (X == 0) are very strange indeed. We don't
4808 // handle them yet except for the trivial case. This could be expanded in the
4809 // future as needed.
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004810
Chris Lattner53e677a2004-04-02 20:23:17 +00004811 // If the value is a constant, check to see if it is known to be non-zero
4812 // already. If so, the backedge will execute zero times.
Dan Gohman622ed672009-05-04 22:02:23 +00004813 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
Nick Lewycky39442af2008-02-21 09:14:53 +00004814 if (!C->getValue()->isNullValue())
Dan Gohmandeff6212010-05-03 22:09:21 +00004815 return getConstant(C->getType(), 0);
Dan Gohman1c343752009-06-27 21:21:31 +00004816 return getCouldNotCompute(); // Otherwise it will loop infinitely.
Chris Lattner53e677a2004-04-02 20:23:17 +00004817 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00004818
Chris Lattner53e677a2004-04-02 20:23:17 +00004819 // We could implement others, but I really doubt anyone writes loops like
4820 // this, and if they did, they would already be constant folded.
Dan Gohman1c343752009-06-27 21:21:31 +00004821 return getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00004822}
4823
Dan Gohmanfd6edef2008-09-15 22:18:04 +00004824/// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB
4825/// (which may not be an immediate predecessor) which has exactly one
4826/// successor from which BB is reachable, or null if no such block is
4827/// found.
4828///
Dan Gohman005752b2010-04-15 16:19:08 +00004829std::pair<BasicBlock *, BasicBlock *>
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004830ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) {
Dan Gohman3d739fe2009-04-30 20:48:53 +00004831 // If the block has a unique predecessor, then there is no path from the
4832 // predecessor to the block that does not go through the direct edge
4833 // from the predecessor to the block.
Dan Gohmanfd6edef2008-09-15 22:18:04 +00004834 if (BasicBlock *Pred = BB->getSinglePredecessor())
Dan Gohman005752b2010-04-15 16:19:08 +00004835 return std::make_pair(Pred, BB);
Dan Gohmanfd6edef2008-09-15 22:18:04 +00004836
4837 // A loop's header is defined to be a block that dominates the loop.
Dan Gohman859b4822009-05-18 15:36:09 +00004838 // If the header has a unique predecessor outside the loop, it must be
4839 // a block that has exactly one successor that can reach the loop.
Dan Gohmanf8a8be82009-04-21 23:15:49 +00004840 if (Loop *L = LI->getLoopFor(BB))
Dan Gohman605c14f2010-06-22 23:43:28 +00004841 return std::make_pair(L->getLoopPredecessor(), L->getHeader());
Dan Gohmanfd6edef2008-09-15 22:18:04 +00004842
Dan Gohman005752b2010-04-15 16:19:08 +00004843 return std::pair<BasicBlock *, BasicBlock *>();
Dan Gohmanfd6edef2008-09-15 22:18:04 +00004844}
4845
Dan Gohman763bad12009-06-20 00:35:32 +00004846/// HasSameValue - SCEV structural equivalence is usually sufficient for
4847/// testing whether two expressions are equal, however for the purposes of
4848/// looking for a condition guarding a loop, it can be useful to be a little
4849/// more general, since a front-end may have replicated the controlling
4850/// expression.
4851///
Dan Gohman0bba49c2009-07-07 17:06:11 +00004852static bool HasSameValue(const SCEV *A, const SCEV *B) {
Dan Gohman763bad12009-06-20 00:35:32 +00004853 // Quick check to see if they are the same SCEV.
4854 if (A == B) return true;
4855
4856 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
4857 // two different instructions with the same value. Check for this case.
4858 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
4859 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
4860 if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
4861 if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
Dan Gohman041de422009-08-25 17:56:57 +00004862 if (AI->isIdenticalTo(BI) && !AI->mayReadFromMemory())
Dan Gohman763bad12009-06-20 00:35:32 +00004863 return true;
4864
4865 // Otherwise assume they may have a different value.
4866 return false;
4867}
4868
Dan Gohmane9796502010-04-24 01:28:42 +00004869/// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with
4870/// predicate Pred. Return true iff any changes were made.
4871///
4872bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred,
4873 const SCEV *&LHS, const SCEV *&RHS) {
4874 bool Changed = false;
4875
4876 // Canonicalize a constant to the right side.
4877 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
4878 // Check for both operands constant.
4879 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
4880 if (ConstantExpr::getICmp(Pred,
4881 LHSC->getValue(),
4882 RHSC->getValue())->isNullValue())
4883 goto trivially_false;
4884 else
4885 goto trivially_true;
4886 }
4887 // Otherwise swap the operands to put the constant on the right.
4888 std::swap(LHS, RHS);
4889 Pred = ICmpInst::getSwappedPredicate(Pred);
4890 Changed = true;
4891 }
4892
4893 // If we're comparing an addrec with a value which is loop-invariant in the
Dan Gohman3abb69c2010-05-03 17:00:11 +00004894 // addrec's loop, put the addrec on the left. Also make a dominance check,
4895 // as both operands could be addrecs loop-invariant in each other's loop.
4896 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) {
4897 const Loop *L = AR->getLoop();
4898 if (LHS->isLoopInvariant(L) && LHS->properlyDominates(L->getHeader(), DT)) {
Dan Gohmane9796502010-04-24 01:28:42 +00004899 std::swap(LHS, RHS);
4900 Pred = ICmpInst::getSwappedPredicate(Pred);
4901 Changed = true;
4902 }
Dan Gohman3abb69c2010-05-03 17:00:11 +00004903 }
Dan Gohmane9796502010-04-24 01:28:42 +00004904
4905 // If there's a constant operand, canonicalize comparisons with boundary
4906 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
4907 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
4908 const APInt &RA = RC->getValue()->getValue();
4909 switch (Pred) {
4910 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!");
4911 case ICmpInst::ICMP_EQ:
4912 case ICmpInst::ICMP_NE:
4913 break;
4914 case ICmpInst::ICMP_UGE:
4915 if ((RA - 1).isMinValue()) {
4916 Pred = ICmpInst::ICMP_NE;
4917 RHS = getConstant(RA - 1);
4918 Changed = true;
4919 break;
4920 }
4921 if (RA.isMaxValue()) {
4922 Pred = ICmpInst::ICMP_EQ;
4923 Changed = true;
4924 break;
4925 }
4926 if (RA.isMinValue()) goto trivially_true;
4927
4928 Pred = ICmpInst::ICMP_UGT;
4929 RHS = getConstant(RA - 1);
4930 Changed = true;
4931 break;
4932 case ICmpInst::ICMP_ULE:
4933 if ((RA + 1).isMaxValue()) {
4934 Pred = ICmpInst::ICMP_NE;
4935 RHS = getConstant(RA + 1);
4936 Changed = true;
4937 break;
4938 }
4939 if (RA.isMinValue()) {
4940 Pred = ICmpInst::ICMP_EQ;
4941 Changed = true;
4942 break;
4943 }
4944 if (RA.isMaxValue()) goto trivially_true;
4945
4946 Pred = ICmpInst::ICMP_ULT;
4947 RHS = getConstant(RA + 1);
4948 Changed = true;
4949 break;
4950 case ICmpInst::ICMP_SGE:
4951 if ((RA - 1).isMinSignedValue()) {
4952 Pred = ICmpInst::ICMP_NE;
4953 RHS = getConstant(RA - 1);
4954 Changed = true;
4955 break;
4956 }
4957 if (RA.isMaxSignedValue()) {
4958 Pred = ICmpInst::ICMP_EQ;
4959 Changed = true;
4960 break;
4961 }
4962 if (RA.isMinSignedValue()) goto trivially_true;
4963
4964 Pred = ICmpInst::ICMP_SGT;
4965 RHS = getConstant(RA - 1);
4966 Changed = true;
4967 break;
4968 case ICmpInst::ICMP_SLE:
4969 if ((RA + 1).isMaxSignedValue()) {
4970 Pred = ICmpInst::ICMP_NE;
4971 RHS = getConstant(RA + 1);
4972 Changed = true;
4973 break;
4974 }
4975 if (RA.isMinSignedValue()) {
4976 Pred = ICmpInst::ICMP_EQ;
4977 Changed = true;
4978 break;
4979 }
4980 if (RA.isMaxSignedValue()) goto trivially_true;
4981
4982 Pred = ICmpInst::ICMP_SLT;
4983 RHS = getConstant(RA + 1);
4984 Changed = true;
4985 break;
4986 case ICmpInst::ICMP_UGT:
4987 if (RA.isMinValue()) {
4988 Pred = ICmpInst::ICMP_NE;
4989 Changed = true;
4990 break;
4991 }
4992 if ((RA + 1).isMaxValue()) {
4993 Pred = ICmpInst::ICMP_EQ;
4994 RHS = getConstant(RA + 1);
4995 Changed = true;
4996 break;
4997 }
4998 if (RA.isMaxValue()) goto trivially_false;
4999 break;
5000 case ICmpInst::ICMP_ULT:
5001 if (RA.isMaxValue()) {
5002 Pred = ICmpInst::ICMP_NE;
5003 Changed = true;
5004 break;
5005 }
5006 if ((RA - 1).isMinValue()) {
5007 Pred = ICmpInst::ICMP_EQ;
5008 RHS = getConstant(RA - 1);
5009 Changed = true;
5010 break;
5011 }
5012 if (RA.isMinValue()) goto trivially_false;
5013 break;
5014 case ICmpInst::ICMP_SGT:
5015 if (RA.isMinSignedValue()) {
5016 Pred = ICmpInst::ICMP_NE;
5017 Changed = true;
5018 break;
5019 }
5020 if ((RA + 1).isMaxSignedValue()) {
5021 Pred = ICmpInst::ICMP_EQ;
5022 RHS = getConstant(RA + 1);
5023 Changed = true;
5024 break;
5025 }
5026 if (RA.isMaxSignedValue()) goto trivially_false;
5027 break;
5028 case ICmpInst::ICMP_SLT:
5029 if (RA.isMaxSignedValue()) {
5030 Pred = ICmpInst::ICMP_NE;
5031 Changed = true;
5032 break;
5033 }
5034 if ((RA - 1).isMinSignedValue()) {
5035 Pred = ICmpInst::ICMP_EQ;
5036 RHS = getConstant(RA - 1);
5037 Changed = true;
5038 break;
5039 }
5040 if (RA.isMinSignedValue()) goto trivially_false;
5041 break;
5042 }
5043 }
5044
5045 // Check for obvious equality.
5046 if (HasSameValue(LHS, RHS)) {
5047 if (ICmpInst::isTrueWhenEqual(Pred))
5048 goto trivially_true;
5049 if (ICmpInst::isFalseWhenEqual(Pred))
5050 goto trivially_false;
5051 }
5052
Dan Gohman03557dc2010-05-03 16:35:17 +00005053 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
5054 // adding or subtracting 1 from one of the operands.
5055 switch (Pred) {
5056 case ICmpInst::ICMP_SLE:
5057 if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) {
5058 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
5059 /*HasNUW=*/false, /*HasNSW=*/true);
5060 Pred = ICmpInst::ICMP_SLT;
5061 Changed = true;
5062 } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) {
Dan Gohmanf16c6802010-05-03 20:23:47 +00005063 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
Dan Gohman03557dc2010-05-03 16:35:17 +00005064 /*HasNUW=*/false, /*HasNSW=*/true);
5065 Pred = ICmpInst::ICMP_SLT;
5066 Changed = true;
5067 }
5068 break;
5069 case ICmpInst::ICMP_SGE:
5070 if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) {
Dan Gohmanf16c6802010-05-03 20:23:47 +00005071 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
Dan Gohman03557dc2010-05-03 16:35:17 +00005072 /*HasNUW=*/false, /*HasNSW=*/true);
5073 Pred = ICmpInst::ICMP_SGT;
5074 Changed = true;
5075 } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) {
5076 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
5077 /*HasNUW=*/false, /*HasNSW=*/true);
5078 Pred = ICmpInst::ICMP_SGT;
5079 Changed = true;
5080 }
5081 break;
5082 case ICmpInst::ICMP_ULE:
5083 if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) {
Dan Gohmanf16c6802010-05-03 20:23:47 +00005084 RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
Dan Gohman03557dc2010-05-03 16:35:17 +00005085 /*HasNUW=*/true, /*HasNSW=*/false);
5086 Pred = ICmpInst::ICMP_ULT;
5087 Changed = true;
5088 } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) {
Dan Gohmanf16c6802010-05-03 20:23:47 +00005089 LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
Dan Gohman03557dc2010-05-03 16:35:17 +00005090 /*HasNUW=*/true, /*HasNSW=*/false);
5091 Pred = ICmpInst::ICMP_ULT;
5092 Changed = true;
5093 }
5094 break;
5095 case ICmpInst::ICMP_UGE:
5096 if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) {
Dan Gohmanf16c6802010-05-03 20:23:47 +00005097 RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
Dan Gohman03557dc2010-05-03 16:35:17 +00005098 /*HasNUW=*/true, /*HasNSW=*/false);
5099 Pred = ICmpInst::ICMP_UGT;
5100 Changed = true;
5101 } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) {
Dan Gohmanf16c6802010-05-03 20:23:47 +00005102 LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
Dan Gohman03557dc2010-05-03 16:35:17 +00005103 /*HasNUW=*/true, /*HasNSW=*/false);
5104 Pred = ICmpInst::ICMP_UGT;
5105 Changed = true;
5106 }
5107 break;
5108 default:
5109 break;
5110 }
5111
Dan Gohmane9796502010-04-24 01:28:42 +00005112 // TODO: More simplifications are possible here.
5113
5114 return Changed;
5115
5116trivially_true:
5117 // Return 0 == 0.
5118 LHS = RHS = getConstant(Type::getInt1Ty(getContext()), 0);
5119 Pred = ICmpInst::ICMP_EQ;
5120 return true;
5121
5122trivially_false:
5123 // Return 0 != 0.
5124 LHS = RHS = getConstant(Type::getInt1Ty(getContext()), 0);
5125 Pred = ICmpInst::ICMP_NE;
5126 return true;
5127}
5128
Dan Gohman85b05a22009-07-13 21:35:55 +00005129bool ScalarEvolution::isKnownNegative(const SCEV *S) {
5130 return getSignedRange(S).getSignedMax().isNegative();
5131}
5132
5133bool ScalarEvolution::isKnownPositive(const SCEV *S) {
5134 return getSignedRange(S).getSignedMin().isStrictlyPositive();
5135}
5136
5137bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
5138 return !getSignedRange(S).getSignedMin().isNegative();
5139}
5140
5141bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
5142 return !getSignedRange(S).getSignedMax().isStrictlyPositive();
5143}
5144
5145bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
5146 return isKnownNegative(S) || isKnownPositive(S);
5147}
5148
5149bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred,
5150 const SCEV *LHS, const SCEV *RHS) {
Dan Gohmand19bba62010-04-24 01:38:36 +00005151 // Canonicalize the inputs first.
5152 (void)SimplifyICmpOperands(Pred, LHS, RHS);
5153
Dan Gohman53c66ea2010-04-11 22:16:48 +00005154 // If LHS or RHS is an addrec, check to see if the condition is true in
5155 // every iteration of the loop.
5156 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
5157 if (isLoopEntryGuardedByCond(
5158 AR->getLoop(), Pred, AR->getStart(), RHS) &&
5159 isLoopBackedgeGuardedByCond(
Dan Gohmanacd8cab2010-05-04 01:12:27 +00005160 AR->getLoop(), Pred, AR->getPostIncExpr(*this), RHS))
Dan Gohman53c66ea2010-04-11 22:16:48 +00005161 return true;
5162 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS))
5163 if (isLoopEntryGuardedByCond(
5164 AR->getLoop(), Pred, LHS, AR->getStart()) &&
5165 isLoopBackedgeGuardedByCond(
Dan Gohmanacd8cab2010-05-04 01:12:27 +00005166 AR->getLoop(), Pred, LHS, AR->getPostIncExpr(*this)))
Dan Gohman53c66ea2010-04-11 22:16:48 +00005167 return true;
Dan Gohman85b05a22009-07-13 21:35:55 +00005168
Dan Gohman53c66ea2010-04-11 22:16:48 +00005169 // Otherwise see what can be done with known constant ranges.
5170 return isKnownPredicateWithRanges(Pred, LHS, RHS);
5171}
5172
5173bool
5174ScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred,
5175 const SCEV *LHS, const SCEV *RHS) {
Dan Gohman85b05a22009-07-13 21:35:55 +00005176 if (HasSameValue(LHS, RHS))
5177 return ICmpInst::isTrueWhenEqual(Pred);
5178
Dan Gohman53c66ea2010-04-11 22:16:48 +00005179 // This code is split out from isKnownPredicate because it is called from
5180 // within isLoopEntryGuardedByCond.
Dan Gohman85b05a22009-07-13 21:35:55 +00005181 switch (Pred) {
5182 default:
Dan Gohman850f7912009-07-16 17:34:36 +00005183 llvm_unreachable("Unexpected ICmpInst::Predicate value!");
Dan Gohman85b05a22009-07-13 21:35:55 +00005184 break;
5185 case ICmpInst::ICMP_SGT:
5186 Pred = ICmpInst::ICMP_SLT;
5187 std::swap(LHS, RHS);
5188 case ICmpInst::ICMP_SLT: {
5189 ConstantRange LHSRange = getSignedRange(LHS);
5190 ConstantRange RHSRange = getSignedRange(RHS);
5191 if (LHSRange.getSignedMax().slt(RHSRange.getSignedMin()))
5192 return true;
5193 if (LHSRange.getSignedMin().sge(RHSRange.getSignedMax()))
5194 return false;
Dan Gohman85b05a22009-07-13 21:35:55 +00005195 break;
5196 }
5197 case ICmpInst::ICMP_SGE:
5198 Pred = ICmpInst::ICMP_SLE;
5199 std::swap(LHS, RHS);
5200 case ICmpInst::ICMP_SLE: {
5201 ConstantRange LHSRange = getSignedRange(LHS);
5202 ConstantRange RHSRange = getSignedRange(RHS);
5203 if (LHSRange.getSignedMax().sle(RHSRange.getSignedMin()))
5204 return true;
5205 if (LHSRange.getSignedMin().sgt(RHSRange.getSignedMax()))
5206 return false;
Dan Gohman85b05a22009-07-13 21:35:55 +00005207 break;
5208 }
5209 case ICmpInst::ICMP_UGT:
5210 Pred = ICmpInst::ICMP_ULT;
5211 std::swap(LHS, RHS);
5212 case ICmpInst::ICMP_ULT: {
5213 ConstantRange LHSRange = getUnsignedRange(LHS);
5214 ConstantRange RHSRange = getUnsignedRange(RHS);
5215 if (LHSRange.getUnsignedMax().ult(RHSRange.getUnsignedMin()))
5216 return true;
5217 if (LHSRange.getUnsignedMin().uge(RHSRange.getUnsignedMax()))
5218 return false;
Dan Gohman85b05a22009-07-13 21:35:55 +00005219 break;
5220 }
5221 case ICmpInst::ICMP_UGE:
5222 Pred = ICmpInst::ICMP_ULE;
5223 std::swap(LHS, RHS);
5224 case ICmpInst::ICMP_ULE: {
5225 ConstantRange LHSRange = getUnsignedRange(LHS);
5226 ConstantRange RHSRange = getUnsignedRange(RHS);
5227 if (LHSRange.getUnsignedMax().ule(RHSRange.getUnsignedMin()))
5228 return true;
5229 if (LHSRange.getUnsignedMin().ugt(RHSRange.getUnsignedMax()))
5230 return false;
Dan Gohman85b05a22009-07-13 21:35:55 +00005231 break;
5232 }
5233 case ICmpInst::ICMP_NE: {
5234 if (getUnsignedRange(LHS).intersectWith(getUnsignedRange(RHS)).isEmptySet())
5235 return true;
5236 if (getSignedRange(LHS).intersectWith(getSignedRange(RHS)).isEmptySet())
5237 return true;
5238
5239 const SCEV *Diff = getMinusSCEV(LHS, RHS);
5240 if (isKnownNonZero(Diff))
5241 return true;
5242 break;
5243 }
5244 case ICmpInst::ICMP_EQ:
Dan Gohmanf117ed42009-07-20 23:54:43 +00005245 // The check at the top of the function catches the case where
5246 // the values are known to be equal.
Dan Gohman85b05a22009-07-13 21:35:55 +00005247 break;
5248 }
5249 return false;
5250}
5251
5252/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
5253/// protected by a conditional between LHS and RHS. This is used to
5254/// to eliminate casts.
5255bool
5256ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
5257 ICmpInst::Predicate Pred,
5258 const SCEV *LHS, const SCEV *RHS) {
5259 // Interpret a null as meaning no loop, where there is obviously no guard
5260 // (interprocedural conditions notwithstanding).
5261 if (!L) return true;
5262
5263 BasicBlock *Latch = L->getLoopLatch();
5264 if (!Latch)
5265 return false;
5266
5267 BranchInst *LoopContinuePredicate =
5268 dyn_cast<BranchInst>(Latch->getTerminator());
5269 if (!LoopContinuePredicate ||
5270 LoopContinuePredicate->isUnconditional())
5271 return false;
5272
Dan Gohmanaf08a362010-08-10 23:46:30 +00005273 return isImpliedCond(Pred, LHS, RHS,
5274 LoopContinuePredicate->getCondition(),
Dan Gohman0f4b2852009-07-21 23:03:19 +00005275 LoopContinuePredicate->getSuccessor(0) != L->getHeader());
Dan Gohman85b05a22009-07-13 21:35:55 +00005276}
5277
Dan Gohman3948d0b2010-04-11 19:27:13 +00005278/// isLoopEntryGuardedByCond - Test whether entry to the loop is protected
Dan Gohman85b05a22009-07-13 21:35:55 +00005279/// by a conditional between LHS and RHS. This is used to help avoid max
5280/// expressions in loop trip counts, and to eliminate casts.
5281bool
Dan Gohman3948d0b2010-04-11 19:27:13 +00005282ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L,
5283 ICmpInst::Predicate Pred,
5284 const SCEV *LHS, const SCEV *RHS) {
Dan Gohman8ea94522009-05-18 16:03:58 +00005285 // Interpret a null as meaning no loop, where there is obviously no guard
5286 // (interprocedural conditions notwithstanding).
5287 if (!L) return false;
5288
Dan Gohman859b4822009-05-18 15:36:09 +00005289 // Starting at the loop predecessor, climb up the predecessor chain, as long
5290 // as there are predecessors that can be found that have unique successors
Dan Gohmanfd6edef2008-09-15 22:18:04 +00005291 // leading to the original header.
Dan Gohman005752b2010-04-15 16:19:08 +00005292 for (std::pair<BasicBlock *, BasicBlock *>
Dan Gohman605c14f2010-06-22 23:43:28 +00005293 Pair(L->getLoopPredecessor(), L->getHeader());
Dan Gohman005752b2010-04-15 16:19:08 +00005294 Pair.first;
5295 Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
Dan Gohman38372182008-08-12 20:17:31 +00005296
5297 BranchInst *LoopEntryPredicate =
Dan Gohman005752b2010-04-15 16:19:08 +00005298 dyn_cast<BranchInst>(Pair.first->getTerminator());
Dan Gohman38372182008-08-12 20:17:31 +00005299 if (!LoopEntryPredicate ||
5300 LoopEntryPredicate->isUnconditional())
5301 continue;
5302
Dan Gohmanaf08a362010-08-10 23:46:30 +00005303 if (isImpliedCond(Pred, LHS, RHS,
5304 LoopEntryPredicate->getCondition(),
Dan Gohman005752b2010-04-15 16:19:08 +00005305 LoopEntryPredicate->getSuccessor(0) != Pair.second))
Dan Gohman38372182008-08-12 20:17:31 +00005306 return true;
Nick Lewycky59cff122008-07-12 07:41:32 +00005307 }
5308
Dan Gohman38372182008-08-12 20:17:31 +00005309 return false;
Nick Lewycky59cff122008-07-12 07:41:32 +00005310}
5311
Dan Gohman0f4b2852009-07-21 23:03:19 +00005312/// isImpliedCond - Test whether the condition described by Pred, LHS,
5313/// and RHS is true whenever the given Cond value evaluates to true.
Dan Gohmanaf08a362010-08-10 23:46:30 +00005314bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred,
Dan Gohman0f4b2852009-07-21 23:03:19 +00005315 const SCEV *LHS, const SCEV *RHS,
Dan Gohmanaf08a362010-08-10 23:46:30 +00005316 Value *FoundCondValue,
Dan Gohman0f4b2852009-07-21 23:03:19 +00005317 bool Inverse) {
Dan Gohman3f46a3a2010-03-01 17:49:51 +00005318 // Recursively handle And and Or conditions.
Dan Gohmanaf08a362010-08-10 23:46:30 +00005319 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) {
Dan Gohman40a5a1b2009-06-24 01:18:18 +00005320 if (BO->getOpcode() == Instruction::And) {
5321 if (!Inverse)
Dan Gohmanaf08a362010-08-10 23:46:30 +00005322 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
5323 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
Dan Gohman40a5a1b2009-06-24 01:18:18 +00005324 } else if (BO->getOpcode() == Instruction::Or) {
5325 if (Inverse)
Dan Gohmanaf08a362010-08-10 23:46:30 +00005326 return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
5327 isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
Dan Gohman40a5a1b2009-06-24 01:18:18 +00005328 }
5329 }
5330
Dan Gohmanaf08a362010-08-10 23:46:30 +00005331 ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
Dan Gohman40a5a1b2009-06-24 01:18:18 +00005332 if (!ICI) return false;
5333
Dan Gohman85b05a22009-07-13 21:35:55 +00005334 // Bail if the ICmp's operands' types are wider than the needed type
5335 // before attempting to call getSCEV on them. This avoids infinite
5336 // recursion, since the analysis of widening casts can require loop
5337 // exit condition information for overflow checking, which would
5338 // lead back here.
5339 if (getTypeSizeInBits(LHS->getType()) <
Dan Gohman0f4b2852009-07-21 23:03:19 +00005340 getTypeSizeInBits(ICI->getOperand(0)->getType()))
Dan Gohman85b05a22009-07-13 21:35:55 +00005341 return false;
5342
Dan Gohman0f4b2852009-07-21 23:03:19 +00005343 // Now that we found a conditional branch that dominates the loop, check to
5344 // see if it is the comparison we are looking for.
5345 ICmpInst::Predicate FoundPred;
5346 if (Inverse)
5347 FoundPred = ICI->getInversePredicate();
5348 else
5349 FoundPred = ICI->getPredicate();
5350
5351 const SCEV *FoundLHS = getSCEV(ICI->getOperand(0));
5352 const SCEV *FoundRHS = getSCEV(ICI->getOperand(1));
Dan Gohman85b05a22009-07-13 21:35:55 +00005353
5354 // Balance the types. The case where FoundLHS' type is wider than
5355 // LHS' type is checked for above.
5356 if (getTypeSizeInBits(LHS->getType()) >
5357 getTypeSizeInBits(FoundLHS->getType())) {
5358 if (CmpInst::isSigned(Pred)) {
5359 FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
5360 FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
5361 } else {
5362 FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType());
5363 FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType());
5364 }
5365 }
5366
Dan Gohman0f4b2852009-07-21 23:03:19 +00005367 // Canonicalize the query to match the way instcombine will have
5368 // canonicalized the comparison.
Dan Gohmand4da5af2010-04-24 01:34:53 +00005369 if (SimplifyICmpOperands(Pred, LHS, RHS))
5370 if (LHS == RHS)
Dan Gohman34c3e362010-05-03 18:00:24 +00005371 return CmpInst::isTrueWhenEqual(Pred);
Dan Gohmand4da5af2010-04-24 01:34:53 +00005372 if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
5373 if (FoundLHS == FoundRHS)
Dan Gohman34c3e362010-05-03 18:00:24 +00005374 return CmpInst::isFalseWhenEqual(Pred);
Dan Gohman0f4b2852009-07-21 23:03:19 +00005375
5376 // Check to see if we can make the LHS or RHS match.
5377 if (LHS == FoundRHS || RHS == FoundLHS) {
5378 if (isa<SCEVConstant>(RHS)) {
5379 std::swap(FoundLHS, FoundRHS);
5380 FoundPred = ICmpInst::getSwappedPredicate(FoundPred);
5381 } else {
5382 std::swap(LHS, RHS);
5383 Pred = ICmpInst::getSwappedPredicate(Pred);
5384 }
5385 }
5386
5387 // Check whether the found predicate is the same as the desired predicate.
5388 if (FoundPred == Pred)
5389 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS);
5390
5391 // Check whether swapping the found predicate makes it the same as the
5392 // desired predicate.
5393 if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) {
5394 if (isa<SCEVConstant>(RHS))
5395 return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS);
5396 else
5397 return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred),
5398 RHS, LHS, FoundLHS, FoundRHS);
5399 }
5400
5401 // Check whether the actual condition is beyond sufficient.
5402 if (FoundPred == ICmpInst::ICMP_EQ)
5403 if (ICmpInst::isTrueWhenEqual(Pred))
5404 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS))
5405 return true;
5406 if (Pred == ICmpInst::ICMP_NE)
5407 if (!ICmpInst::isTrueWhenEqual(FoundPred))
5408 if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS))
5409 return true;
5410
5411 // Otherwise assume the worst.
5412 return false;
Dan Gohman85b05a22009-07-13 21:35:55 +00005413}
5414
Dan Gohman0f4b2852009-07-21 23:03:19 +00005415/// isImpliedCondOperands - Test whether the condition described by Pred,
Dan Gohman3f46a3a2010-03-01 17:49:51 +00005416/// LHS, and RHS is true whenever the condition described by Pred, FoundLHS,
Dan Gohman0f4b2852009-07-21 23:03:19 +00005417/// and FoundRHS is true.
5418bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred,
5419 const SCEV *LHS, const SCEV *RHS,
5420 const SCEV *FoundLHS,
5421 const SCEV *FoundRHS) {
5422 return isImpliedCondOperandsHelper(Pred, LHS, RHS,
5423 FoundLHS, FoundRHS) ||
5424 // ~x < ~y --> x > y
5425 isImpliedCondOperandsHelper(Pred, LHS, RHS,
5426 getNotSCEV(FoundRHS),
5427 getNotSCEV(FoundLHS));
5428}
5429
5430/// isImpliedCondOperandsHelper - Test whether the condition described by
Dan Gohman3f46a3a2010-03-01 17:49:51 +00005431/// Pred, LHS, and RHS is true whenever the condition described by Pred,
Dan Gohman0f4b2852009-07-21 23:03:19 +00005432/// FoundLHS, and FoundRHS is true.
Dan Gohman85b05a22009-07-13 21:35:55 +00005433bool
Dan Gohman0f4b2852009-07-21 23:03:19 +00005434ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred,
5435 const SCEV *LHS, const SCEV *RHS,
5436 const SCEV *FoundLHS,
5437 const SCEV *FoundRHS) {
Dan Gohman85b05a22009-07-13 21:35:55 +00005438 switch (Pred) {
Dan Gohman850f7912009-07-16 17:34:36 +00005439 default: llvm_unreachable("Unexpected ICmpInst::Predicate value!");
5440 case ICmpInst::ICMP_EQ:
5441 case ICmpInst::ICMP_NE:
5442 if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS))
5443 return true;
5444 break;
Dan Gohman85b05a22009-07-13 21:35:55 +00005445 case ICmpInst::ICMP_SLT:
Dan Gohman850f7912009-07-16 17:34:36 +00005446 case ICmpInst::ICMP_SLE:
Dan Gohman53c66ea2010-04-11 22:16:48 +00005447 if (isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, LHS, FoundLHS) &&
5448 isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, RHS, FoundRHS))
Dan Gohman85b05a22009-07-13 21:35:55 +00005449 return true;
5450 break;
5451 case ICmpInst::ICMP_SGT:
Dan Gohman850f7912009-07-16 17:34:36 +00005452 case ICmpInst::ICMP_SGE:
Dan Gohman53c66ea2010-04-11 22:16:48 +00005453 if (isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, LHS, FoundLHS) &&
5454 isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, RHS, FoundRHS))
Dan Gohman85b05a22009-07-13 21:35:55 +00005455 return true;
5456 break;
5457 case ICmpInst::ICMP_ULT:
Dan Gohman850f7912009-07-16 17:34:36 +00005458 case ICmpInst::ICMP_ULE:
Dan Gohman53c66ea2010-04-11 22:16:48 +00005459 if (isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, LHS, FoundLHS) &&
5460 isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, RHS, FoundRHS))
Dan Gohman85b05a22009-07-13 21:35:55 +00005461 return true;
5462 break;
5463 case ICmpInst::ICMP_UGT:
Dan Gohman850f7912009-07-16 17:34:36 +00005464 case ICmpInst::ICMP_UGE:
Dan Gohman53c66ea2010-04-11 22:16:48 +00005465 if (isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, LHS, FoundLHS) &&
5466 isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, RHS, FoundRHS))
Dan Gohman85b05a22009-07-13 21:35:55 +00005467 return true;
5468 break;
5469 }
5470
5471 return false;
Dan Gohman40a5a1b2009-06-24 01:18:18 +00005472}
5473
Dan Gohman51f53b72009-06-21 23:46:38 +00005474/// getBECount - Subtract the end and start values and divide by the step,
5475/// rounding up, to get the number of times the backedge is executed. Return
5476/// CouldNotCompute if an intermediate computation overflows.
Dan Gohman0bba49c2009-07-07 17:06:11 +00005477const SCEV *ScalarEvolution::getBECount(const SCEV *Start,
Dan Gohmanf5074ec2009-07-13 22:05:32 +00005478 const SCEV *End,
Dan Gohman1f96e672009-09-17 18:05:20 +00005479 const SCEV *Step,
5480 bool NoWrap) {
Dan Gohman52fddd32010-01-26 04:40:18 +00005481 assert(!isKnownNegative(Step) &&
5482 "This code doesn't handle negative strides yet!");
5483
Dan Gohman51f53b72009-06-21 23:46:38 +00005484 const Type *Ty = Start->getType();
Dan Gohmandeff6212010-05-03 22:09:21 +00005485 const SCEV *NegOne = getConstant(Ty, (uint64_t)-1);
Dan Gohman0bba49c2009-07-07 17:06:11 +00005486 const SCEV *Diff = getMinusSCEV(End, Start);
5487 const SCEV *RoundUp = getAddExpr(Step, NegOne);
Dan Gohman51f53b72009-06-21 23:46:38 +00005488
5489 // Add an adjustment to the difference between End and Start so that
5490 // the division will effectively round up.
Dan Gohman0bba49c2009-07-07 17:06:11 +00005491 const SCEV *Add = getAddExpr(Diff, RoundUp);
Dan Gohman51f53b72009-06-21 23:46:38 +00005492
Dan Gohman1f96e672009-09-17 18:05:20 +00005493 if (!NoWrap) {
5494 // Check Add for unsigned overflow.
5495 // TODO: More sophisticated things could be done here.
5496 const Type *WideTy = IntegerType::get(getContext(),
5497 getTypeSizeInBits(Ty) + 1);
5498 const SCEV *EDiff = getZeroExtendExpr(Diff, WideTy);
5499 const SCEV *ERoundUp = getZeroExtendExpr(RoundUp, WideTy);
5500 const SCEV *OperandExtendedAdd = getAddExpr(EDiff, ERoundUp);
5501 if (getZeroExtendExpr(Add, WideTy) != OperandExtendedAdd)
5502 return getCouldNotCompute();
5503 }
Dan Gohman51f53b72009-06-21 23:46:38 +00005504
5505 return getUDivExpr(Add, Step);
5506}
5507
Chris Lattnerdb25de42005-08-15 23:33:51 +00005508/// HowManyLessThans - Return the number of times a backedge containing the
5509/// specified less-than comparison will execute. If not computable, return
Dan Gohman86fbf2f2009-06-06 14:37:11 +00005510/// CouldNotCompute.
Dan Gohman64a845e2009-06-24 04:48:43 +00005511ScalarEvolution::BackedgeTakenInfo
5512ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS,
5513 const Loop *L, bool isSigned) {
Chris Lattnerdb25de42005-08-15 23:33:51 +00005514 // Only handle: "ADDREC < LoopInvariant".
Dan Gohman1c343752009-06-27 21:21:31 +00005515 if (!RHS->isLoopInvariant(L)) return getCouldNotCompute();
Chris Lattnerdb25de42005-08-15 23:33:51 +00005516
Dan Gohman35738ac2009-05-04 22:30:44 +00005517 const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS);
Chris Lattnerdb25de42005-08-15 23:33:51 +00005518 if (!AddRec || AddRec->getLoop() != L)
Dan Gohman1c343752009-06-27 21:21:31 +00005519 return getCouldNotCompute();
Chris Lattnerdb25de42005-08-15 23:33:51 +00005520
Dan Gohman1f96e672009-09-17 18:05:20 +00005521 // Check to see if we have a flag which makes analysis easy.
5522 bool NoWrap = isSigned ? AddRec->hasNoSignedWrap() :
5523 AddRec->hasNoUnsignedWrap();
5524
Chris Lattnerdb25de42005-08-15 23:33:51 +00005525 if (AddRec->isAffine()) {
Dan Gohmana1af7572009-04-30 20:47:05 +00005526 unsigned BitWidth = getTypeSizeInBits(AddRec->getType());
Dan Gohman0bba49c2009-07-07 17:06:11 +00005527 const SCEV *Step = AddRec->getStepRecurrence(*this);
Dan Gohmana1af7572009-04-30 20:47:05 +00005528
Dan Gohman52fddd32010-01-26 04:40:18 +00005529 if (Step->isZero())
Dan Gohman1c343752009-06-27 21:21:31 +00005530 return getCouldNotCompute();
Dan Gohman52fddd32010-01-26 04:40:18 +00005531 if (Step->isOne()) {
Dan Gohmana1af7572009-04-30 20:47:05 +00005532 // With unit stride, the iteration never steps past the limit value.
Dan Gohman52fddd32010-01-26 04:40:18 +00005533 } else if (isKnownPositive(Step)) {
Dan Gohmanf451cb82010-02-10 16:03:48 +00005534 // Test whether a positive iteration can step past the limit
Dan Gohman52fddd32010-01-26 04:40:18 +00005535 // value and past the maximum value for its type in a single step.
5536 // Note that it's not sufficient to check NoWrap here, because even
5537 // though the value after a wrap is undefined, it's not undefined
5538 // behavior, so if wrap does occur, the loop could either terminate or
Dan Gohman155eec72010-01-26 18:32:54 +00005539 // loop infinitely, but in either case, the loop is guaranteed to
Dan Gohman52fddd32010-01-26 04:40:18 +00005540 // iterate at least until the iteration where the wrapping occurs.
Dan Gohmandeff6212010-05-03 22:09:21 +00005541 const SCEV *One = getConstant(Step->getType(), 1);
Dan Gohman52fddd32010-01-26 04:40:18 +00005542 if (isSigned) {
5543 APInt Max = APInt::getSignedMaxValue(BitWidth);
5544 if ((Max - getSignedRange(getMinusSCEV(Step, One)).getSignedMax())
5545 .slt(getSignedRange(RHS).getSignedMax()))
5546 return getCouldNotCompute();
5547 } else {
5548 APInt Max = APInt::getMaxValue(BitWidth);
5549 if ((Max - getUnsignedRange(getMinusSCEV(Step, One)).getUnsignedMax())
5550 .ult(getUnsignedRange(RHS).getUnsignedMax()))
5551 return getCouldNotCompute();
5552 }
Dan Gohmana1af7572009-04-30 20:47:05 +00005553 } else
Dan Gohman52fddd32010-01-26 04:40:18 +00005554 // TODO: Handle negative strides here and below.
Dan Gohman1c343752009-06-27 21:21:31 +00005555 return getCouldNotCompute();
Chris Lattnerdb25de42005-08-15 23:33:51 +00005556
Dan Gohmana1af7572009-04-30 20:47:05 +00005557 // We know the LHS is of the form {n,+,s} and the RHS is some loop-invariant
5558 // m. So, we count the number of iterations in which {n,+,s} < m is true.
5559 // Note that we cannot simply return max(m-n,0)/s because it's not safe to
Wojciech Matyjewicza65ee032008-02-13 12:21:32 +00005560 // treat m-n as signed nor unsigned due to overflow possibility.
Chris Lattnerdb25de42005-08-15 23:33:51 +00005561
Wojciech Matyjewicz3a4cbe22008-02-13 11:51:34 +00005562 // First, we get the value of the LHS in the first iteration: n
Dan Gohman0bba49c2009-07-07 17:06:11 +00005563 const SCEV *Start = AddRec->getOperand(0);
Wojciech Matyjewicz3a4cbe22008-02-13 11:51:34 +00005564
Dan Gohmana1af7572009-04-30 20:47:05 +00005565 // Determine the minimum constant start value.
Dan Gohman85b05a22009-07-13 21:35:55 +00005566 const SCEV *MinStart = getConstant(isSigned ?
5567 getSignedRange(Start).getSignedMin() :
5568 getUnsignedRange(Start).getUnsignedMin());
Wojciech Matyjewicz3a4cbe22008-02-13 11:51:34 +00005569
Dan Gohmana1af7572009-04-30 20:47:05 +00005570 // If we know that the condition is true in order to enter the loop,
5571 // then we know that it will run exactly (m-n)/s times. Otherwise, we
Dan Gohman6c0866c2009-05-24 23:45:28 +00005572 // only know that it will execute (max(m,n)-n)/s times. In both cases,
5573 // the division must round up.
Dan Gohman0bba49c2009-07-07 17:06:11 +00005574 const SCEV *End = RHS;
Dan Gohman3948d0b2010-04-11 19:27:13 +00005575 if (!isLoopEntryGuardedByCond(L,
5576 isSigned ? ICmpInst::ICMP_SLT :
5577 ICmpInst::ICMP_ULT,
5578 getMinusSCEV(Start, Step), RHS))
Dan Gohmana1af7572009-04-30 20:47:05 +00005579 End = isSigned ? getSMaxExpr(RHS, Start)
5580 : getUMaxExpr(RHS, Start);
5581
5582 // Determine the maximum constant end value.
Dan Gohman85b05a22009-07-13 21:35:55 +00005583 const SCEV *MaxEnd = getConstant(isSigned ?
5584 getSignedRange(End).getSignedMax() :
5585 getUnsignedRange(End).getUnsignedMax());
Dan Gohmana1af7572009-04-30 20:47:05 +00005586
Dan Gohman52fddd32010-01-26 04:40:18 +00005587 // If MaxEnd is within a step of the maximum integer value in its type,
5588 // adjust it down to the minimum value which would produce the same effect.
Dan Gohman3f46a3a2010-03-01 17:49:51 +00005589 // This allows the subsequent ceiling division of (N+(step-1))/step to
Dan Gohman52fddd32010-01-26 04:40:18 +00005590 // compute the correct value.
5591 const SCEV *StepMinusOne = getMinusSCEV(Step,
Dan Gohmandeff6212010-05-03 22:09:21 +00005592 getConstant(Step->getType(), 1));
Dan Gohman52fddd32010-01-26 04:40:18 +00005593 MaxEnd = isSigned ?
5594 getSMinExpr(MaxEnd,
5595 getMinusSCEV(getConstant(APInt::getSignedMaxValue(BitWidth)),
5596 StepMinusOne)) :
5597 getUMinExpr(MaxEnd,
5598 getMinusSCEV(getConstant(APInt::getMaxValue(BitWidth)),
5599 StepMinusOne));
5600
Dan Gohmana1af7572009-04-30 20:47:05 +00005601 // Finally, we subtract these two values and divide, rounding up, to get
5602 // the number of times the backedge is executed.
Dan Gohman1f96e672009-09-17 18:05:20 +00005603 const SCEV *BECount = getBECount(Start, End, Step, NoWrap);
Dan Gohmana1af7572009-04-30 20:47:05 +00005604
5605 // The maximum backedge count is similar, except using the minimum start
5606 // value and the maximum end value.
Dan Gohman1f96e672009-09-17 18:05:20 +00005607 const SCEV *MaxBECount = getBECount(MinStart, MaxEnd, Step, NoWrap);
Dan Gohmana1af7572009-04-30 20:47:05 +00005608
5609 return BackedgeTakenInfo(BECount, MaxBECount);
Chris Lattnerdb25de42005-08-15 23:33:51 +00005610 }
5611
Dan Gohman1c343752009-06-27 21:21:31 +00005612 return getCouldNotCompute();
Chris Lattnerdb25de42005-08-15 23:33:51 +00005613}
5614
Chris Lattner53e677a2004-04-02 20:23:17 +00005615/// getNumIterationsInRange - Return the number of iterations of this loop that
5616/// produce values in the specified constant range. Another way of looking at
5617/// this is that it returns the first iteration number where the value is not in
5618/// the condition, thus computing the exit count. If the iteration count can't
5619/// be computed, an instance of SCEVCouldNotCompute is returned.
Dan Gohman0bba49c2009-07-07 17:06:11 +00005620const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range,
Dan Gohman64a845e2009-06-24 04:48:43 +00005621 ScalarEvolution &SE) const {
Chris Lattner53e677a2004-04-02 20:23:17 +00005622 if (Range.isFullSet()) // Infinite loop.
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00005623 return SE.getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00005624
5625 // If the start is a non-zero constant, shift the range to simplify things.
Dan Gohman622ed672009-05-04 22:02:23 +00005626 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart()))
Reid Spencercae57542007-03-02 00:28:52 +00005627 if (!SC->getValue()->isZero()) {
Dan Gohman0bba49c2009-07-07 17:06:11 +00005628 SmallVector<const SCEV *, 4> Operands(op_begin(), op_end());
Dan Gohmandeff6212010-05-03 22:09:21 +00005629 Operands[0] = SE.getConstant(SC->getType(), 0);
Dan Gohman0bba49c2009-07-07 17:06:11 +00005630 const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop());
Dan Gohman622ed672009-05-04 22:02:23 +00005631 if (const SCEVAddRecExpr *ShiftedAddRec =
5632 dyn_cast<SCEVAddRecExpr>(Shifted))
Chris Lattner53e677a2004-04-02 20:23:17 +00005633 return ShiftedAddRec->getNumIterationsInRange(
Dan Gohman246b2562007-10-22 18:31:58 +00005634 Range.subtract(SC->getValue()->getValue()), SE);
Chris Lattner53e677a2004-04-02 20:23:17 +00005635 // This is strange and shouldn't happen.
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00005636 return SE.getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00005637 }
5638
5639 // The only time we can solve this is when we have all constant indices.
5640 // Otherwise, we cannot determine the overflow conditions.
5641 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
5642 if (!isa<SCEVConstant>(getOperand(i)))
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00005643 return SE.getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00005644
5645
5646 // Okay at this point we know that all elements of the chrec are constants and
5647 // that the start element is zero.
5648
5649 // First check to see if the range contains zero. If not, the first
5650 // iteration exits.
Dan Gohmanaf79fb52009-04-21 01:07:12 +00005651 unsigned BitWidth = SE.getTypeSizeInBits(getType());
Dan Gohman2d1be872009-04-16 03:18:22 +00005652 if (!Range.contains(APInt(BitWidth, 0)))
Dan Gohmandeff6212010-05-03 22:09:21 +00005653 return SE.getConstant(getType(), 0);
Misha Brukman2b37d7c2005-04-21 21:13:18 +00005654
Chris Lattner53e677a2004-04-02 20:23:17 +00005655 if (isAffine()) {
5656 // If this is an affine expression then we have this situation:
5657 // Solve {0,+,A} in Range === Ax in Range
5658
Nick Lewyckyeefdebe2007-07-16 02:08:00 +00005659 // We know that zero is in the range. If A is positive then we know that
5660 // the upper value of the range must be the first possible exit value.
5661 // If A is negative then the lower of the range is the last possible loop
5662 // value. Also note that we already checked for a full range.
Dan Gohman2d1be872009-04-16 03:18:22 +00005663 APInt One(BitWidth,1);
Nick Lewyckyeefdebe2007-07-16 02:08:00 +00005664 APInt A = cast<SCEVConstant>(getOperand(1))->getValue()->getValue();
5665 APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower();
Chris Lattner53e677a2004-04-02 20:23:17 +00005666
Nick Lewyckyeefdebe2007-07-16 02:08:00 +00005667 // The exit value should be (End+A)/A.
Nick Lewycky9a2f9312007-09-27 14:12:54 +00005668 APInt ExitVal = (End + A).udiv(A);
Owen Andersoneed707b2009-07-24 23:12:02 +00005669 ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal);
Chris Lattner53e677a2004-04-02 20:23:17 +00005670
5671 // Evaluate at the exit value. If we really did fall out of the valid
5672 // range, then we computed our trip count, otherwise wrap around or other
5673 // things must have happened.
Dan Gohman246b2562007-10-22 18:31:58 +00005674 ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE);
Reid Spencera6e8a952007-03-01 07:54:15 +00005675 if (Range.contains(Val->getValue()))
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00005676 return SE.getCouldNotCompute(); // Something strange happened
Chris Lattner53e677a2004-04-02 20:23:17 +00005677
5678 // Ensure that the previous value is in the range. This is a sanity check.
Reid Spencer581b0d42007-02-28 19:57:34 +00005679 assert(Range.contains(
Dan Gohman64a845e2009-06-24 04:48:43 +00005680 EvaluateConstantChrecAtConstant(this,
Owen Andersoneed707b2009-07-24 23:12:02 +00005681 ConstantInt::get(SE.getContext(), ExitVal - One), SE)->getValue()) &&
Chris Lattner53e677a2004-04-02 20:23:17 +00005682 "Linear scev computation is off in a bad way!");
Dan Gohman246b2562007-10-22 18:31:58 +00005683 return SE.getConstant(ExitValue);
Chris Lattner53e677a2004-04-02 20:23:17 +00005684 } else if (isQuadratic()) {
5685 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of the
5686 // quadratic equation to solve it. To do this, we must frame our problem in
5687 // terms of figuring out when zero is crossed, instead of when
5688 // Range.getUpper() is crossed.
Dan Gohman0bba49c2009-07-07 17:06:11 +00005689 SmallVector<const SCEV *, 4> NewOps(op_begin(), op_end());
Dan Gohman246b2562007-10-22 18:31:58 +00005690 NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper()));
Dan Gohman0bba49c2009-07-07 17:06:11 +00005691 const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop());
Chris Lattner53e677a2004-04-02 20:23:17 +00005692
5693 // Next, solve the constructed addrec
Dan Gohman0bba49c2009-07-07 17:06:11 +00005694 std::pair<const SCEV *,const SCEV *> Roots =
Dan Gohman246b2562007-10-22 18:31:58 +00005695 SolveQuadraticEquation(cast<SCEVAddRecExpr>(NewAddRec), SE);
Dan Gohman35738ac2009-05-04 22:30:44 +00005696 const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first);
5697 const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second);
Chris Lattner53e677a2004-04-02 20:23:17 +00005698 if (R1) {
5699 // Pick the smallest positive root value.
Zhou Sheng6b6b6ef2007-01-11 12:24:14 +00005700 if (ConstantInt *CB =
Owen Andersonbaf3c402009-07-29 18:55:55 +00005701 dyn_cast<ConstantInt>(ConstantExpr::getICmp(ICmpInst::ICMP_ULT,
Owen Anderson76f600b2009-07-06 22:37:39 +00005702 R1->getValue(), R2->getValue()))) {
Reid Spencer579dca12007-01-12 04:24:46 +00005703 if (CB->getZExtValue() == false)
Chris Lattner53e677a2004-04-02 20:23:17 +00005704 std::swap(R1, R2); // R1 is the minimum root now.
Misha Brukman2b37d7c2005-04-21 21:13:18 +00005705
Chris Lattner53e677a2004-04-02 20:23:17 +00005706 // Make sure the root is not off by one. The returned iteration should
5707 // not be in the range, but the previous one should be. When solving
5708 // for "X*X < 5", for example, we should not return a root of 2.
5709 ConstantInt *R1Val = EvaluateConstantChrecAtConstant(this,
Dan Gohman246b2562007-10-22 18:31:58 +00005710 R1->getValue(),
5711 SE);
Reid Spencera6e8a952007-03-01 07:54:15 +00005712 if (Range.contains(R1Val->getValue())) {
Chris Lattner53e677a2004-04-02 20:23:17 +00005713 // The next iteration must be out of the range...
Owen Anderson76f600b2009-07-06 22:37:39 +00005714 ConstantInt *NextVal =
Owen Andersoneed707b2009-07-24 23:12:02 +00005715 ConstantInt::get(SE.getContext(), R1->getValue()->getValue()+1);
Misha Brukman2b37d7c2005-04-21 21:13:18 +00005716
Dan Gohman246b2562007-10-22 18:31:58 +00005717 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE);
Reid Spencera6e8a952007-03-01 07:54:15 +00005718 if (!Range.contains(R1Val->getValue()))
Dan Gohman246b2562007-10-22 18:31:58 +00005719 return SE.getConstant(NextVal);
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00005720 return SE.getCouldNotCompute(); // Something strange happened
Chris Lattner53e677a2004-04-02 20:23:17 +00005721 }
Misha Brukman2b37d7c2005-04-21 21:13:18 +00005722
Chris Lattner53e677a2004-04-02 20:23:17 +00005723 // If R1 was not in the range, then it is a good return value. Make
5724 // sure that R1-1 WAS in the range though, just in case.
Owen Anderson76f600b2009-07-06 22:37:39 +00005725 ConstantInt *NextVal =
Owen Andersoneed707b2009-07-24 23:12:02 +00005726 ConstantInt::get(SE.getContext(), R1->getValue()->getValue()-1);
Dan Gohman246b2562007-10-22 18:31:58 +00005727 R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE);
Reid Spencera6e8a952007-03-01 07:54:15 +00005728 if (Range.contains(R1Val->getValue()))
Chris Lattner53e677a2004-04-02 20:23:17 +00005729 return R1;
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00005730 return SE.getCouldNotCompute(); // Something strange happened
Chris Lattner53e677a2004-04-02 20:23:17 +00005731 }
5732 }
5733 }
5734
Dan Gohmanf4ccfcb2009-04-18 17:58:19 +00005735 return SE.getCouldNotCompute();
Chris Lattner53e677a2004-04-02 20:23:17 +00005736}
5737
5738
5739
5740//===----------------------------------------------------------------------===//
Dan Gohman35738ac2009-05-04 22:30:44 +00005741// SCEVCallbackVH Class Implementation
5742//===----------------------------------------------------------------------===//
5743
Dan Gohman1959b752009-05-19 19:22:47 +00005744void ScalarEvolution::SCEVCallbackVH::deleted() {
Dan Gohmanddf9f992009-07-13 22:20:53 +00005745 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
Dan Gohman35738ac2009-05-04 22:30:44 +00005746 if (PHINode *PN = dyn_cast<PHINode>(getValPtr()))
5747 SE->ConstantEvolutionLoopExitValue.erase(PN);
5748 SE->Scalars.erase(getValPtr());
5749 // this now dangles!
5750}
5751
Dan Gohman81f91212010-07-28 01:09:07 +00005752void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
Dan Gohmanddf9f992009-07-13 22:20:53 +00005753 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
Eric Christophere6cbfa62010-07-29 01:25:38 +00005754
Dan Gohman35738ac2009-05-04 22:30:44 +00005755 // Forget all the expressions associated with users of the old value,
5756 // so that future queries will recompute the expressions using the new
5757 // value.
Dan Gohmanab37f502010-08-02 23:49:30 +00005758 Value *Old = getValPtr();
Dan Gohman35738ac2009-05-04 22:30:44 +00005759 SmallVector<User *, 16> Worklist;
Dan Gohman69fcae92009-07-14 14:34:04 +00005760 SmallPtrSet<User *, 8> Visited;
Dan Gohman35738ac2009-05-04 22:30:44 +00005761 for (Value::use_iterator UI = Old->use_begin(), UE = Old->use_end();
5762 UI != UE; ++UI)
5763 Worklist.push_back(*UI);
5764 while (!Worklist.empty()) {
5765 User *U = Worklist.pop_back_val();
5766 // Deleting the Old value will cause this to dangle. Postpone
5767 // that until everything else is done.
Dan Gohman59846ac2010-07-28 00:28:25 +00005768 if (U == Old)
Dan Gohman35738ac2009-05-04 22:30:44 +00005769 continue;
Dan Gohman69fcae92009-07-14 14:34:04 +00005770 if (!Visited.insert(U))
5771 continue;
Dan Gohman35738ac2009-05-04 22:30:44 +00005772 if (PHINode *PN = dyn_cast<PHINode>(U))
5773 SE->ConstantEvolutionLoopExitValue.erase(PN);
Dan Gohman69fcae92009-07-14 14:34:04 +00005774 SE->Scalars.erase(U);
5775 for (Value::use_iterator UI = U->use_begin(), UE = U->use_end();
5776 UI != UE; ++UI)
5777 Worklist.push_back(*UI);
Dan Gohman35738ac2009-05-04 22:30:44 +00005778 }
Dan Gohman59846ac2010-07-28 00:28:25 +00005779 // Delete the Old value.
5780 if (PHINode *PN = dyn_cast<PHINode>(Old))
5781 SE->ConstantEvolutionLoopExitValue.erase(PN);
5782 SE->Scalars.erase(Old);
5783 // this now dangles!
Dan Gohman35738ac2009-05-04 22:30:44 +00005784}
5785
Dan Gohman1959b752009-05-19 19:22:47 +00005786ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
Dan Gohman35738ac2009-05-04 22:30:44 +00005787 : CallbackVH(V), SE(se) {}
5788
5789//===----------------------------------------------------------------------===//
Chris Lattner53e677a2004-04-02 20:23:17 +00005790// ScalarEvolution Class Implementation
5791//===----------------------------------------------------------------------===//
5792
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005793ScalarEvolution::ScalarEvolution()
Owen Anderson90c579d2010-08-06 18:33:48 +00005794 : FunctionPass(ID), FirstUnknown(0) {
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005795}
5796
Chris Lattner53e677a2004-04-02 20:23:17 +00005797bool ScalarEvolution::runOnFunction(Function &F) {
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005798 this->F = &F;
5799 LI = &getAnalysis<LoopInfo>();
5800 TD = getAnalysisIfAvailable<TargetData>();
Dan Gohman454d26d2010-02-22 04:11:59 +00005801 DT = &getAnalysis<DominatorTree>();
Chris Lattner53e677a2004-04-02 20:23:17 +00005802 return false;
5803}
5804
5805void ScalarEvolution::releaseMemory() {
Dan Gohmanab37f502010-08-02 23:49:30 +00005806 // Iterate through all the SCEVUnknown instances and call their
5807 // destructors, so that they release their references to their values.
5808 for (SCEVUnknown *U = FirstUnknown; U; U = U->Next)
5809 U->~SCEVUnknown();
5810 FirstUnknown = 0;
5811
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005812 Scalars.clear();
5813 BackedgeTakenCounts.clear();
5814 ConstantEvolutionLoopExitValue.clear();
Dan Gohman6bce6432009-05-08 20:47:27 +00005815 ValuesAtScopes.clear();
Dan Gohman1c343752009-06-27 21:21:31 +00005816 UniqueSCEVs.clear();
5817 SCEVAllocator.Reset();
Chris Lattner53e677a2004-04-02 20:23:17 +00005818}
5819
5820void ScalarEvolution::getAnalysisUsage(AnalysisUsage &AU) const {
5821 AU.setPreservesAll();
Chris Lattner53e677a2004-04-02 20:23:17 +00005822 AU.addRequiredTransitive<LoopInfo>();
Dan Gohman1cd92752010-01-19 22:21:27 +00005823 AU.addRequiredTransitive<DominatorTree>();
Dan Gohman2d1be872009-04-16 03:18:22 +00005824}
5825
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005826bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
Dan Gohman46bdfb02009-02-24 18:55:53 +00005827 return !isa<SCEVCouldNotCompute>(getBackedgeTakenCount(L));
Chris Lattner53e677a2004-04-02 20:23:17 +00005828}
5829
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005830static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
Chris Lattner53e677a2004-04-02 20:23:17 +00005831 const Loop *L) {
5832 // Print all inner loops first
5833 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
5834 PrintLoopInfo(OS, SE, *I);
Misha Brukman2b37d7c2005-04-21 21:13:18 +00005835
Dan Gohman30733292010-01-09 18:17:45 +00005836 OS << "Loop ";
5837 WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false);
5838 OS << ": ";
Chris Lattnerf1ab4b42004-04-18 22:14:10 +00005839
Dan Gohman5d984912009-12-18 01:14:11 +00005840 SmallVector<BasicBlock *, 8> ExitBlocks;
Chris Lattnerf1ab4b42004-04-18 22:14:10 +00005841 L->getExitBlocks(ExitBlocks);
5842 if (ExitBlocks.size() != 1)
Nick Lewyckyaeb5e5c2008-01-02 02:49:20 +00005843 OS << "<multiple exits> ";
Chris Lattner53e677a2004-04-02 20:23:17 +00005844
Dan Gohman46bdfb02009-02-24 18:55:53 +00005845 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
5846 OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L);
Chris Lattner53e677a2004-04-02 20:23:17 +00005847 } else {
Dan Gohman46bdfb02009-02-24 18:55:53 +00005848 OS << "Unpredictable backedge-taken count. ";
Chris Lattner53e677a2004-04-02 20:23:17 +00005849 }
5850
Dan Gohman30733292010-01-09 18:17:45 +00005851 OS << "\n"
5852 "Loop ";
5853 WriteAsOperand(OS, L->getHeader(), /*PrintType=*/false);
5854 OS << ": ";
Dan Gohmanaa551ae2009-06-24 00:33:16 +00005855
5856 if (!isa<SCEVCouldNotCompute>(SE->getMaxBackedgeTakenCount(L))) {
5857 OS << "max backedge-taken count is " << *SE->getMaxBackedgeTakenCount(L);
5858 } else {
5859 OS << "Unpredictable max backedge-taken count. ";
5860 }
5861
5862 OS << "\n";
Chris Lattner53e677a2004-04-02 20:23:17 +00005863}
5864
Dan Gohman5d984912009-12-18 01:14:11 +00005865void ScalarEvolution::print(raw_ostream &OS, const Module *) const {
Dan Gohman3f46a3a2010-03-01 17:49:51 +00005866 // ScalarEvolution's implementation of the print method is to print
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005867 // out SCEV values of all instructions that are interesting. Doing
5868 // this potentially causes it to create new SCEV objects though,
5869 // which technically conflicts with the const qualifier. This isn't
Dan Gohman1afdc5f2009-07-10 20:25:29 +00005870 // observable from outside the class though, so casting away the
5871 // const isn't dangerous.
Dan Gohman5d984912009-12-18 01:14:11 +00005872 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
Chris Lattner53e677a2004-04-02 20:23:17 +00005873
Dan Gohman30733292010-01-09 18:17:45 +00005874 OS << "Classifying expressions for: ";
5875 WriteAsOperand(OS, F, /*PrintType=*/false);
5876 OS << "\n";
Chris Lattner53e677a2004-04-02 20:23:17 +00005877 for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I)
Dan Gohmana189bae2010-05-03 17:03:23 +00005878 if (isSCEVable(I->getType()) && !isa<CmpInst>(*I)) {
Dan Gohmanc902e132009-07-13 23:03:05 +00005879 OS << *I << '\n';
Dan Gohman8dae1382008-09-14 17:21:12 +00005880 OS << " --> ";
Dan Gohman0bba49c2009-07-07 17:06:11 +00005881 const SCEV *SV = SE.getSCEV(&*I);
Chris Lattner53e677a2004-04-02 20:23:17 +00005882 SV->print(OS);
Misha Brukman2b37d7c2005-04-21 21:13:18 +00005883
Dan Gohman0c689c52009-06-19 17:49:54 +00005884 const Loop *L = LI->getLoopFor((*I).getParent());
5885
Dan Gohman0bba49c2009-07-07 17:06:11 +00005886 const SCEV *AtUse = SE.getSCEVAtScope(SV, L);
Dan Gohman0c689c52009-06-19 17:49:54 +00005887 if (AtUse != SV) {
5888 OS << " --> ";
5889 AtUse->print(OS);
5890 }
5891
5892 if (L) {
Dan Gohman9e7d9882009-06-18 00:37:45 +00005893 OS << "\t\t" "Exits: ";
Dan Gohman0bba49c2009-07-07 17:06:11 +00005894 const SCEV *ExitValue = SE.getSCEVAtScope(SV, L->getParentLoop());
Dan Gohmand594e6f2009-05-24 23:25:42 +00005895 if (!ExitValue->isLoopInvariant(L)) {
Chris Lattner53e677a2004-04-02 20:23:17 +00005896 OS << "<<Unknown>>";
5897 } else {
5898 OS << *ExitValue;
5899 }
5900 }
5901
Chris Lattner53e677a2004-04-02 20:23:17 +00005902 OS << "\n";
5903 }
5904
Dan Gohman30733292010-01-09 18:17:45 +00005905 OS << "Determining loop execution counts for: ";
5906 WriteAsOperand(OS, F, /*PrintType=*/false);
5907 OS << "\n";
Dan Gohmanf8a8be82009-04-21 23:15:49 +00005908 for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I)
5909 PrintLoopInfo(OS, &SE, *I);
Chris Lattner53e677a2004-04-02 20:23:17 +00005910}
Dan Gohmanb7ef7292009-04-21 00:47:46 +00005911