Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / llvm / 839a0159c44e64503720b424dc386e4732f56ad0 / . / lib / Transforms / Scalar / LoopStrengthReduce.cpp
blob: 6456ca1eb0c147d74dbe7ecf4e9ce65801745d01 [file] [log] [blame]
//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This transformation analyzes and transforms the induction variables (and
// computations derived from them) into forms suitable for efficient execution
// on the target.
//
// This pass performs a strength reduction on array references inside loops that
// have as one or more of their components the loop induction variable, it
// rewrites expressions to take advantage of scaled-index addressing modes
// available on the target, and it performs a variety of other optimizations
// related to loop induction variables.
//
// Terminology note: this code has a lot of handling for "post-increment" or
// "post-inc" users. This is not talking about post-increment addressing modes;
// it is instead talking about code like this:
//
// %i = phi [ 0, %entry ], [ %i.next, %latch ]
// ...
// %i.next = add %i, 1
// %c = icmp eq %i.next, %n
//
// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
// it's useful to think about these as the same register, with some uses using
// the value of the register before the add and some using // it after. In this
// example, the icmp is a post-increment user, since it uses %i.next, which is
// the value of the induction variable after the increment. The other common
// case of post-increment users is users outside the loop.
//
// TODO: More sophistication in the way Formulae are generated.
//
// TODO: Handle multiple loops at a time.
//
// TODO: test/CodeGen/X86/full-lsr.ll should get full lsr. The problem is
// that {0,+,1}<%bb> is getting picked first because all 7 uses can
// use it, and while it's a pretty good solution, it means that LSR
// doesn't look further to find an even better solution.
//
// TODO: Should TargetLowering::AddrMode::BaseGV be changed to a ConstantExpr
// instead of a GlobalValue?
//
// TODO: When truncation is free, truncate ICmp users' operands to make it a
// smaller encoding (on x86 at least).
//
// TODO: When a negated register is used by an add (such as in a list of
// multiple base registers, or as the increment expression in an addrec),
// we may not actually need both reg and (-1 * reg) in registers; the
// negation can be implemented by using a sub instead of an add. The
// lack of support for taking this into consideration when making
// register pressure decisions is partly worked around by the "Special"
// use kind.
//
//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "loop-reduce"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Constants.h"
#include "llvm/Instructions.h"
#include "llvm/IntrinsicInst.h"
#include "llvm/DerivedTypes.h"
#include "llvm/Analysis/IVUsers.h"
#include "llvm/Analysis/Dominators.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ValueHandle.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetLowering.h"
#include <algorithm>
using namespace llvm;
namespace {
// Constant strides come first which in turns are sorted by their absolute
// values. If absolute values are the same, then positive strides comes first.
// e.g.
// 4, -1, X, 1, 2 ==> 1, -1, 2, 4, X
struct StrideCompare {
const ScalarEvolution &SE;
explicit StrideCompare(const ScalarEvolution &se) : SE(se) {}
bool operator()(const SCEV *const &LHS, const SCEV *const &RHS) const {
const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS);
const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS);
if (LHSC && RHSC) {
unsigned BitWidth = std::max(SE.getTypeSizeInBits(LHS->getType()),
SE.getTypeSizeInBits(RHS->getType()));
APInt LV = LHSC->getValue()->getValue();
APInt RV = RHSC->getValue()->getValue();
LV.sextOrTrunc(BitWidth);
RV.sextOrTrunc(BitWidth);
APInt ALV = LV.abs();
APInt ARV = RV.abs();
if (ALV == ARV) {
if (LV != RV)
return LV.sgt(RV);
} else {
return ALV.ult(ARV);
}
// If it's the same value but different type, sort by bit width so
// that we emit larger induction variables before smaller
// ones, letting the smaller be re-written in terms of larger ones.
return SE.getTypeSizeInBits(RHS->getType()) <
SE.getTypeSizeInBits(LHS->getType());
}
return LHSC && !RHSC;
}
};
/// RegSortData - This class holds data which is used to order reuse
/// candidates.
class RegSortData {
public:
/// Bits - This represents the set of LSRUses (by index) which reference a
/// particular register.
SmallBitVector Bits;
/// MaxComplexity - This represents the greatest complexity (see the comments
/// on Formula::getComplexity) seen with a particular register.
uint32_t MaxComplexity;
/// Index - This holds an arbitrary value used as a last-resort tie breaker
/// to ensure deterministic behavior.
unsigned Index;
RegSortData() {}
void print(raw_ostream &OS) const;
void dump() const;
};
}
void RegSortData::print(raw_ostream &OS) const {
OS << "[NumUses=" << Bits.count()
<< ", MaxComplexity=";
OS.write_hex(MaxComplexity);
OS << ", Index=" << Index << ']';
}
void RegSortData::dump() const {
print(errs()); errs() << '\n';
}
namespace {
/// RegCount - This is a helper class to sort a given set of registers
/// according to associated RegSortData values.
class RegCount {
public:
const SCEV *Reg;
RegSortData Sort;
RegCount(const SCEV *R, const RegSortData &RSD)
: Reg(R), Sort(RSD) {}
// Sort by count. Returning true means the register is preferred.
bool operator<(const RegCount &Other) const {
// Sort by the number of unique uses of this register.
unsigned A = Sort.Bits.count();
unsigned B = Other.Sort.Bits.count();
if (A != B) return A > B;
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) {
const SCEVAddRecExpr *BR = dyn_cast<SCEVAddRecExpr>(Other.Reg);
// AddRecs have higher priority than other things.
if (!BR) return true;
// Prefer affine values.
if (AR->isAffine() != BR->isAffine())
return AR->isAffine();
const Loop *AL = AR->getLoop();
const Loop *BL = BR->getLoop();
if (AL != BL) {
unsigned ADepth = AL->getLoopDepth();
unsigned BDepth = BL->getLoopDepth();
// Prefer a less deeply nested addrec.
if (ADepth != BDepth)
return ADepth < BDepth;
// Different loops at the same depth; do something arbitrary.
BasicBlock *AH = AL->getHeader();
BasicBlock *BH = BL->getHeader();
for (Function::iterator I = AH, E = AH->getParent()->end(); I != E; ++I)
if (&*I == BH) return true;
return false;
}
// Sort addrecs by stride.
const SCEV *AStep = AR->getOperand(1);
const SCEV *BStep = BR->getOperand(1);
if (AStep != BStep) {
if (const SCEVConstant *AC = dyn_cast<SCEVConstant>(AStep)) {
const SCEVConstant *BC = dyn_cast<SCEVConstant>(BStep);
if (!BC) return true;
// Arbitrarily prefer wider registers.
if (AC->getValue()->getValue().getBitWidth() !=
BC->getValue()->getValue().getBitWidth())
return AC->getValue()->getValue().getBitWidth() >
BC->getValue()->getValue().getBitWidth();
// Ignore the sign bit, assuming that striding by a negative value
// is just as easy as by a positive value.
// Prefer the addrec with the lesser absolute value stride, as it
// will allow uses to have simpler addressing modes.
return AC->getValue()->getValue().abs()
.ult(BC->getValue()->getValue().abs());
}
}
// Then sort by the register which will permit the simplest uses.
// This is a heuristic; currently we only track the most complex use as a
// representative.
if (Sort.MaxComplexity != Other.Sort.MaxComplexity)
return Sort.MaxComplexity < Other.Sort.MaxComplexity;
// Then sort them by their start values.
const SCEV *AStart = AR->getStart();
const SCEV *BStart = BR->getStart();
if (AStart != BStart) {
if (const SCEVConstant *AC = dyn_cast<SCEVConstant>(AStart)) {
const SCEVConstant *BC = dyn_cast<SCEVConstant>(BStart);
if (!BC) return true;
// Arbitrarily prefer wider registers.
if (AC->getValue()->getValue().getBitWidth() !=
BC->getValue()->getValue().getBitWidth())
return AC->getValue()->getValue().getBitWidth() >
BC->getValue()->getValue().getBitWidth();
// Prefer positive over negative if the absolute values are the same.
if (AC->getValue()->getValue().abs() ==
BC->getValue()->getValue().abs())
return AC->getValue()->getValue().isStrictlyPositive();
// Prefer the addrec with the lesser absolute value start.
return AC->getValue()->getValue().abs()
.ult(BC->getValue()->getValue().abs());
}
}
} else {
// AddRecs have higher priority than other things.
if (isa<SCEVAddRecExpr>(Other.Reg)) return false;
// Sort by the register which will permit the simplest uses.
// This is a heuristic; currently we only track the most complex use as a
// representative.
if (Sort.MaxComplexity != Other.Sort.MaxComplexity)
return Sort.MaxComplexity < Other.Sort.MaxComplexity;
}
// Tie-breaker: the arbitrary index, to ensure a reliable ordering.
return Sort.Index < Other.Sort.Index;
}
void print(raw_ostream &OS) const;
void dump() const;
};
}
void RegCount::print(raw_ostream &OS) const {
OS << *Reg << ':';
Sort.print(OS);
}
void RegCount::dump() const {
print(errs()); errs() << '\n';
}
namespace {
/// Formula - This class holds information that describes a formula for
/// satisfying a use. It may include broken-out immediates and scaled registers.
struct Formula {
/// AM - This is used to represent complex addressing, as well as other kinds
/// of interesting uses.
TargetLowering::AddrMode AM;
/// BaseRegs - The list of "base" registers for this use. When this is
/// non-empty, AM.HasBaseReg should be set to true.
SmallVector<const SCEV *, 2> BaseRegs;
/// ScaledReg - The 'scaled' register for this use. This should be non-null
/// when AM.Scale is not zero.
const SCEV *ScaledReg;
Formula() : ScaledReg(0) {}
unsigned getNumRegs() const;
uint32_t getComplexity() const;
const Type *getType() const;
void InitialMatch(const SCEV *S, Loop *L,
ScalarEvolution &SE, DominatorTree &DT);
/// referencesReg - Test if this formula references the given register.
bool referencesReg(const SCEV *S) const {
return S == ScaledReg ||
std::find(BaseRegs.begin(), BaseRegs.end(), S) != BaseRegs.end();
}
bool operator==(const Formula &Other) const {
return BaseRegs == Other.BaseRegs &&
ScaledReg == Other.ScaledReg &&
AM.Scale == Other.AM.Scale &&
AM.BaseOffs == Other.AM.BaseOffs &&
AM.BaseGV == Other.AM.BaseGV;
}
// This sorts at least partially based on host pointer values which are
// not deterministic, so it is only usable for uniqification.
bool operator<(const Formula &Other) const {
if (BaseRegs != Other.BaseRegs)
return BaseRegs < Other.BaseRegs;
if (ScaledReg != Other.ScaledReg)
return ScaledReg < Other.ScaledReg;
if (AM.Scale != Other.AM.Scale)
return AM.Scale < Other.AM.Scale;
if (AM.BaseOffs != Other.AM.BaseOffs)
return AM.BaseOffs < Other.AM.BaseOffs;
if (AM.BaseGV != Other.AM.BaseGV)
return AM.BaseGV < Other.AM.BaseGV;
return false;
}
void print(raw_ostream &OS) const;
void dump() const;
};
}
/// getNumRegs - Return the total number of register operands used by this
/// formula.
unsigned Formula::getNumRegs() const {
return !!ScaledReg + BaseRegs.size();
}
/// getComplexity - Return an oversimplified value indicating the complexity
/// of this formula. This is used as a tie-breaker in choosing register
/// preferences.
uint32_t Formula::getComplexity() const {
// Encode the information in a uint32_t so that comparing with operator<
// will be interesting.
return
// Most significant, the number of registers. This saturates because we
// need the bits, and because beyond a few hundred it doesn't really matter.
(std::min(getNumRegs(), (1u<<15)-1) << 17) |
// Having multiple base regs is worse than having a base reg and a scale.
((BaseRegs.size() > 1) << 16) |
// Scale absolute value.
((AM.Scale != 0 ? (Log2_64(abs64(AM.Scale)) + 1) : 0u) << 9) |
// Scale sign, which is less significant than the absolute value.
((AM.Scale < 0) << 8) |
// Offset absolute value.
((AM.BaseOffs != 0 ? (Log2_64(abs64(AM.BaseOffs)) + 1) : 0u) << 1) |
// If a GV is present, treat it like a maximal offset.
((AM.BaseGV ? ((1u<<7)-1) : 0) << 1) |
// Offset sign, which is less significant than the absolute offset.
((AM.BaseOffs < 0) << 0);
}
/// getType - Return the type of this formula, if it has one, or null
/// otherwise. This type is meaningless except for the bit size.
const Type *Formula::getType() const {
return !BaseRegs.empty() ? BaseRegs.front()->getType() :
ScaledReg ? ScaledReg->getType() :
AM.BaseGV ? AM.BaseGV->getType() :
0;
}
namespace {
/// ComplexitySorter - A predicate which orders Formulae by the number of
/// registers they contain.
struct ComplexitySorter {
bool operator()(const Formula &LHS, const Formula &RHS) const {
unsigned L = LHS.getNumRegs();
unsigned R = RHS.getNumRegs();
if (L != R) return L < R;
return LHS.getComplexity() < RHS.getComplexity();
}
};
}
/// DoInitialMatch - Recurrsion helper for InitialMatch.
static void DoInitialMatch(const SCEV *S, Loop *L,
SmallVectorImpl<const SCEV *> &Good,
SmallVectorImpl<const SCEV *> &Bad,
ScalarEvolution &SE, DominatorTree &DT) {
// Collect expressions which properly dominate the loop header.
if (S->properlyDominates(L->getHeader(), &DT)) {
Good.push_back(S);
return;
}
// Look at add operands.
if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
I != E; ++I)
DoInitialMatch(*I, L, Good, Bad, SE, DT);
return;
}
// Look at addrec operands.
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
if (!AR->getStart()->isZero()) {
DoInitialMatch(AR->getStart(), L, Good, Bad, SE, DT);
DoInitialMatch(SE.getAddRecExpr(SE.getIntegerSCEV(0, AR->getType()),
AR->getStepRecurrence(SE),
AR->getLoop()),
L, Good, Bad, SE, DT);
return;
}
}
// Handle a multiplication by -1 (negation) if it didn't fold.
if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S))
if (Mul->getOperand(0)->isAllOnesValue()) {
SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end());
const SCEV *NewMul = SE.getMulExpr(Ops);
SmallVector<const SCEV *, 4> MyGood;
SmallVector<const SCEV *, 4> MyBad;
DoInitialMatch(NewMul, L, MyGood, MyBad, SE, DT);
const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue(
SE.getEffectiveSCEVType(NewMul->getType())));
for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(),
E = MyGood.end(); I != E; ++I)
Good.push_back(SE.getMulExpr(NegOne, *I));
for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(),
E = MyBad.end(); I != E; ++I)
Bad.push_back(SE.getMulExpr(NegOne, *I));
return;
}
// Ok, we can't do anything interesting. Just stuff the whole thing into a
// register and hope for the best.
Bad.push_back(S);
}
/// InitialMatch - Incorporate loop-variant parts of S into this Formula,
/// attempting to keep all loop-invariant and loop-computable values in a
/// single base register.
void Formula::InitialMatch(const SCEV *S, Loop *L,
ScalarEvolution &SE, DominatorTree &DT) {
SmallVector<const SCEV *, 4> Good;
SmallVector<const SCEV *, 4> Bad;
DoInitialMatch(S, L, Good, Bad, SE, DT);
if (!Good.empty()) {
BaseRegs.push_back(SE.getAddExpr(Good));
AM.HasBaseReg = true;
}
if (!Bad.empty()) {
BaseRegs.push_back(SE.getAddExpr(Bad));
AM.HasBaseReg = true;
}
}
void Formula::print(raw_ostream &OS) const {
bool First = true;
if (AM.BaseGV) {
if (!First) OS << " + "; else First = false;
WriteAsOperand(OS, AM.BaseGV, /*PrintType=*/false);
}
if (AM.BaseOffs != 0) {
if (!First) OS << " + "; else First = false;
OS << AM.BaseOffs;
}
for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(),
E = BaseRegs.end(); I != E; ++I) {
if (!First) OS << " + "; else First = false;
OS << "reg(";
OS << **I;
OS << ")";
}
if (AM.Scale != 0) {
if (!First) OS << " + "; else First = false;
OS << AM.Scale << "*reg(";
if (ScaledReg)
OS << *ScaledReg;
else
OS << "<unknown>";
OS << ")";
}
}
void Formula::dump() const {
print(errs()); errs() << '\n';
}
/// getSDiv - Return an expression for LHS /s RHS, if it can be determined,
/// or null otherwise. If IgnoreSignificantBits is true, expressions like
/// (X * Y) /s Y are simplified to Y, ignoring that the multiplication may
/// overflow, which is useful when the result will be used in a context where
/// the most significant bits are ignored.
static const SCEV *getSDiv(const SCEV *LHS, const SCEV *RHS,
ScalarEvolution &SE,
bool IgnoreSignificantBits = false) {
// Handle the trivial case, which works for any SCEV type.
if (LHS == RHS)
return SE.getIntegerSCEV(1, LHS->getType());
// Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do some
// folding.
if (RHS->isAllOnesValue())
return SE.getMulExpr(LHS, RHS);
// Check for a division of a constant by a constant.
if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) {
const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS);
if (!RC)
return 0;
if (C->getValue()->getValue().srem(RC->getValue()->getValue()) != 0)
return 0;
return SE.getConstant(C->getValue()->getValue()
.sdiv(RC->getValue()->getValue()));
}
// Distribute the sdiv over addrec operands.
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) {
const SCEV *Start = getSDiv(AR->getStart(), RHS, SE,
IgnoreSignificantBits);
if (!Start) return 0;
const SCEV *Step = getSDiv(AR->getStepRecurrence(SE), RHS, SE,
IgnoreSignificantBits);
if (!Step) return 0;
return SE.getAddRecExpr(Start, Step, AR->getLoop());
}
// Distribute the sdiv over add operands.
if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) {
SmallVector<const SCEV *, 8> Ops;
for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
I != E; ++I) {
const SCEV *Op = getSDiv(*I, RHS, SE,
IgnoreSignificantBits);
if (!Op) return 0;
Ops.push_back(Op);
}
return SE.getAddExpr(Ops);
}
// Check for a multiply operand that we can pull RHS out of.
if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS))
if (IgnoreSignificantBits || Mul->hasNoSignedWrap()) {
SmallVector<const SCEV *, 4> Ops;
bool Found = false;
for (SCEVMulExpr::op_iterator I = Mul->op_begin(), E = Mul->op_end();
I != E; ++I) {
if (!Found)
if (const SCEV *Q = getSDiv(*I, RHS, SE, IgnoreSignificantBits)) {
Ops.push_back(Q);
Found = true;
continue;
}
Ops.push_back(*I);
}
return Found ? SE.getMulExpr(Ops) : 0;
}
// Otherwise we don't know.
return 0;
}
namespace {
/// LSRUse - This class holds the state that LSR keeps for each use in
/// IVUsers, as well as uses invented by LSR itself. It includes information
/// about what kinds of things can be folded into the user, information
/// about the user itself, and information about how the use may be satisfied.
/// TODO: Represent multiple users of the same expression in common?
class LSRUse {
SmallSet<Formula, 8> FormulaeUniquifier;
public:
/// KindType - An enum for a kind of use, indicating what types of
/// scaled and immediate operands it might support.
enum KindType {
Basic, ///< A normal use, with no folding.
Special, ///< A special case of basic, allowing -1 scales.
Address, ///< An address use; folding according to TargetLowering
ICmpZero ///< An equality icmp with both operands folded into one.
// TODO: Add a generic icmp too?
};
KindType Kind;
const Type *AccessTy;
Instruction *UserInst;
Value *OperandValToReplace;
/// PostIncLoop - If this user is to use the post-incremented value of an
/// induction variable, this variable is non-null and holds the loop
/// associated with the induction variable.
const Loop *PostIncLoop;
/// Formulae - A list of ways to build a value that can satisfy this user.
/// After the list is populated, one of these is selected heuristically and
/// used to formulate a replacement for OperandValToReplace in UserInst.
SmallVector<Formula, 12> Formulae;
LSRUse() : Kind(Basic), AccessTy(0),
UserInst(0), OperandValToReplace(0), PostIncLoop(0) {}
void InsertInitialFormula(const SCEV *S, Loop *L,
ScalarEvolution &SE, DominatorTree &DT);
void InsertSupplementalFormula(const SCEV *S);
bool InsertFormula(const Formula &F);
void Rewrite(Loop *L, SCEVExpander &Rewriter,
SmallVectorImpl<WeakVH> &DeadInsts,
ScalarEvolution &SE, DominatorTree &DT,
Pass *P) const;
void print(raw_ostream &OS) const;
void dump() const;
private:
Value *Expand(BasicBlock::iterator IP,
Loop *L, SCEVExpander &Rewriter,
SmallVectorImpl<WeakVH> &DeadInsts,
ScalarEvolution &SE, DominatorTree &DT) const;
};
}
/// ExtractImmediate - If S involves the addition of a constant integer value,
/// return that integer value, and mutate S to point to a new SCEV with that
/// value excluded.
static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
if (C->getValue()->getValue().getMinSignedBits() <= 64) {
S = SE.getIntegerSCEV(0, C->getType());
return C->getValue()->getSExtValue();
}
} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end());
int64_t Result = ExtractImmediate(NewOps.front(), SE);
S = SE.getAddExpr(NewOps);
return Result;
} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end());
int64_t Result = ExtractImmediate(NewOps.front(), SE);
S = SE.getAddRecExpr(NewOps, AR->getLoop());
return Result;
}
return 0;
}
/// ExtractSymbol - If S involves the addition of a GlobalValue address,
/// return that symbol, and mutate S to point to a new SCEV with that
/// value excluded.
static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) {
if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) {
S = SE.getIntegerSCEV(0, GV->getType());
return GV;
}
} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end());
GlobalValue *Result = ExtractSymbol(NewOps.back(), SE);
S = SE.getAddExpr(NewOps);
return Result;
} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end());
GlobalValue *Result = ExtractSymbol(NewOps.front(), SE);
S = SE.getAddRecExpr(NewOps, AR->getLoop());
return Result;
}
return 0;
}
/// isLegalUse - Test whether the use described by AM is "legal", meaning
/// it can be completely folded into the user instruction at isel time.
/// This includes address-mode folding and special icmp tricks.
static bool isLegalUse(const TargetLowering::AddrMode &AM,
LSRUse::KindType Kind, const Type *AccessTy,
const TargetLowering *TLI) {
switch (Kind) {
case LSRUse::Address:
// If we have low-level target information, ask the target if it can
// completely fold this address.
if (TLI) return TLI->isLegalAddressingMode(AM, AccessTy);
// Otherwise, just guess that reg+reg addressing is legal.
return !AM.BaseGV && AM.BaseOffs == 0 && AM.Scale <= 1;
case LSRUse::ICmpZero:
// There's not even a target hook for querying whether it would be legal
// to fold a GV into an ICmp.
if (AM.BaseGV)
return false;
// ICmp only has two operands; don't allow more than two non-trivial parts.
if (AM.Scale != 0 && AM.HasBaseReg && AM.BaseOffs != 0)
return false;
// ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale
// by putting the scaled register in the other operand of the icmp.
if (AM.Scale != 0 && AM.Scale != -1)
return false;
// If we have low-level target information, ask the target if it can
// fold an integer immediate on an icmp.
if (AM.BaseOffs != 0) {
if (TLI) return TLI->isLegalICmpImmediate(-AM.BaseOffs);
return false;
}
return true;
case LSRUse::Basic:
// Only handle single-register values.
return !AM.BaseGV && AM.Scale == 0 && AM.BaseOffs == 0;
case LSRUse::Special:
// Only handle -1 scales, or no scale.
return AM.Scale == 0 || AM.Scale == -1;
}
return false;
}
static bool isAlwaysFoldable(const SCEV *S,
bool HasBaseReg,
LSRUse::KindType Kind, const Type *AccessTy,
const TargetLowering *TLI,
ScalarEvolution &SE) {
// Fast-path: zero is always foldable.
if (S->isZero()) return true;
// Conservatively, create an address with an immediate and a
// base and a scale.
TargetLowering::AddrMode AM;
AM.BaseOffs = ExtractImmediate(S, SE);
AM.BaseGV = ExtractSymbol(S, SE);
AM.HasBaseReg = HasBaseReg;
AM.Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
// If there's anything else involved, it's not foldable.
if (!S->isZero()) return false;
return isLegalUse(AM, Kind, AccessTy, TLI);
}
/// InsertFormula - If the given formula has not yet been inserted, add it
/// to the list, and return true. Return false otherwise.
bool LSRUse::InsertFormula(const Formula &F) {
Formula Copy = F;
// Sort the base regs, to avoid adding the same solution twice with
// the base regs in different orders. This uses host pointer values, but
// it doesn't matter since it's only used for uniquifying.
std::sort(Copy.BaseRegs.begin(), Copy.BaseRegs.end());
DEBUG(for (SmallVectorImpl<const SCEV *>::const_iterator I =
F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I)
assert(!(*I)->isZero() && "Zero allocated in a base register!");
assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&
"Zero allocated in a scaled register!"));
if (FormulaeUniquifier.insert(Copy)) {
Formulae.push_back(F);
return true;
}
return false;
}
void
LSRUse::InsertInitialFormula(const SCEV *S, Loop *L,
ScalarEvolution &SE, DominatorTree &DT) {
Formula F;
F.InitialMatch(S, L, SE, DT);
bool Inserted = InsertFormula(F);
assert(Inserted && "Initial formula already exists!"); (void)Inserted;
}
void
LSRUse::InsertSupplementalFormula(const SCEV *S) {
Formula F;
F.BaseRegs.push_back(S);
F.AM.HasBaseReg = true;
bool Inserted = InsertFormula(F);
assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
}
/// getImmediateDominator - A handy utility for the specific DominatorTree
/// query that we need here.
///
static BasicBlock *getImmediateDominator(BasicBlock *BB, DominatorTree &DT) {
DomTreeNode *Node = DT.getNode(BB);
if (!Node) return 0;
Node = Node->getIDom();
if (!Node) return 0;
return Node->getBlock();
}
Value *LSRUse::Expand(BasicBlock::iterator IP,
Loop *L, SCEVExpander &Rewriter,
SmallVectorImpl<WeakVH> &DeadInsts,
ScalarEvolution &SE, DominatorTree &DT) const {
// Then, collect some instructions which we will remain dominated by when
// expanding the replacement. These must be dominated by any operands that
// will be required in the expansion.
SmallVector<Instruction *, 4> Inputs;
if (Instruction *I = dyn_cast<Instruction>(OperandValToReplace))
Inputs.push_back(I);
if (Kind == ICmpZero)
if (Instruction *I =
dyn_cast<Instruction>(cast<ICmpInst>(UserInst)->getOperand(1)))
Inputs.push_back(I);
if (PostIncLoop && !L->contains(UserInst))
Inputs.push_back(L->getLoopLatch()->getTerminator());
// Then, climb up the immediate dominator tree as far as we can go while
// still being dominated by the input positions.
for (;;) {
bool AllDominate = true;
Instruction *BetterPos = 0;
BasicBlock *IDom = getImmediateDominator(IP->getParent(), DT);
if (!IDom) break;
Instruction *Tentative = IDom->getTerminator();
for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(),
E = Inputs.end(); I != E; ++I) {
Instruction *Inst = *I;
if (Inst == Tentative || !DT.dominates(Inst, Tentative)) {
AllDominate = false;
break;
}
if (IDom == Inst->getParent() &&
(!BetterPos || DT.dominates(BetterPos, Inst)))
BetterPos = next(BasicBlock::iterator(Inst));
}
if (!AllDominate)
break;
if (BetterPos)
IP = BetterPos;
else
IP = Tentative;
}
while (isa<PHINode>(IP)) ++IP;
// The first formula in the list is the winner.
const Formula &F = Formulae.front();
// Inform the Rewriter if we have a post-increment use, so that it can
// perform an advantageous expansion.
Rewriter.setPostInc(PostIncLoop);
// This is the type that the user actually needs.
const Type *OpTy = OperandValToReplace->getType();
// This will be the type that we'll initially expand to.
const Type *Ty = F.getType();
if (!Ty)
// No type known; just expand directly to the ultimate type.
Ty = OpTy;
else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy))
// Expand directly to the ultimate type if it's the right size.
Ty = OpTy;
// This is the type to do integer arithmetic in.
const Type *IntTy = SE.getEffectiveSCEVType(Ty);
// Build up a list of operands to add together to form the full base.
SmallVector<const SCEV *, 8> Ops;
// Expand the BaseRegs portion.
for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
E = F.BaseRegs.end(); I != E; ++I) {
const SCEV *Reg = *I;
assert(!Reg->isZero() && "Zero allocated in a base register!");
// If we're expanding for a post-inc user for the add-rec's loop, make the
// post-inc adjustment.
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg))
if (AR->getLoop() == PostIncLoop)
Reg = SE.getAddExpr(Reg, AR->getStepRecurrence(SE));
Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, 0, IP)));
}
// Expand the ScaledReg portion.
Value *ICmpScaledV = 0;
if (F.AM.Scale != 0) {
const SCEV *ScaledS = F.ScaledReg;
// If we're expanding for a post-inc user for the add-rec's loop, make the
// post-inc adjustment.
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(ScaledS))
if (AR->getLoop() == PostIncLoop)
ScaledS = SE.getAddExpr(ScaledS, AR->getStepRecurrence(SE));
if (Kind == ICmpZero) {
// An interesting way of "folding" with an icmp is to use a negated
// scale, which we'll implement by inserting it into the other operand
// of the icmp.
assert(F.AM.Scale == -1 &&
"The only scale supported by ICmpZero uses is -1!");
ICmpScaledV = Rewriter.expandCodeFor(ScaledS, 0, IP);
} else {
// Otherwise just expand the scaled register and an explicit scale,
// which is expected to be matched as part of the address.
ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, 0, IP));
const Type *ScaledTy = SE.getEffectiveSCEVType(ScaledS->getType());
ScaledS = SE.getMulExpr(ScaledS,
SE.getSCEV(ConstantInt::get(ScaledTy,
F.AM.Scale)));
Ops.push_back(ScaledS);
}
}
// Expand the immediate portions.
if (F.AM.BaseGV)
Ops.push_back(SE.getSCEV(F.AM.BaseGV));
if (F.AM.BaseOffs != 0) {
if (Kind == ICmpZero) {
// The other interesting way of "folding" with an ICmpZero is to use a
// negated immediate.
if (!ICmpScaledV)
ICmpScaledV = ConstantInt::get(IntTy, -F.AM.BaseOffs);
else {
Ops.push_back(SE.getUnknown(ICmpScaledV));
ICmpScaledV = ConstantInt::get(IntTy, F.AM.BaseOffs);
}
} else {
// Just add the immediate values. These again are expected to be matched
// as part of the address.
Ops.push_back(SE.getSCEV(ConstantInt::get(IntTy, F.AM.BaseOffs)));
}
}
// Emit instructions summing all the operands.
const SCEV *FullS = Ops.empty() ?
SE.getIntegerSCEV(0, IntTy) :
SE.getAddExpr(Ops);
Value *FullV = Rewriter.expandCodeFor(FullS, Ty, IP);
// We're done expanding now, so reset the rewriter.
Rewriter.setPostInc(0);
// An ICmpZero Formula represents an ICmp which we're handling as a
// comparison against zero. Now that we've expanded an expression for that
// form, update the ICmp's other operand.
if (Kind == ICmpZero) {
ICmpInst *CI = cast<ICmpInst>(UserInst);
DeadInsts.push_back(CI->getOperand(1));
assert(!F.AM.BaseGV && "ICmp does not support folding a global value and "
"a scale at the same time!");
if (F.AM.Scale == -1) {
if (ICmpScaledV->getType() != OpTy) {
Instruction *Cast =
CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false,
OpTy, false),
ICmpScaledV, OpTy, "tmp", CI);
ICmpScaledV = Cast;
}
CI->setOperand(1, ICmpScaledV);
} else {
assert(F.AM.Scale == 0 &&
"ICmp does not support folding a global value and "
"a scale at the same time!");
Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy),
-(uint64_t)F.AM.BaseOffs);
if (C->getType() != OpTy)
C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
OpTy, false),
C, OpTy);
CI->setOperand(1, C);
}
}
return FullV;
}
/// Rewrite - Emit instructions for the leading candidate expression for this
/// LSRUse (this is called "expanding"), and update the UserInst to reference
/// the newly expanded value.
void LSRUse::Rewrite(Loop *L, SCEVExpander &Rewriter,
SmallVectorImpl<WeakVH> &DeadInsts,
ScalarEvolution &SE, DominatorTree &DT,
Pass *P) const {
const Type *OpTy = OperandValToReplace->getType();
// First, find an insertion point that dominates UserInst. For PHI nodes,
// find the nearest block which dominates all the relevant uses.
if (PHINode *PN = dyn_cast<PHINode>(UserInst)) {
DenseMap<BasicBlock *, Value *> Inserted;
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
if (PN->getIncomingValue(i) == OperandValToReplace) {
BasicBlock *BB = PN->getIncomingBlock(i);
// If this is a critical edge, split the edge so that we do not insert
// the code on all predecessor/successor paths. We do this unless this
// is the canonical backedge for this loop, which complicates post-inc
// users.
if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 &&
!isa<IndirectBrInst>(BB->getTerminator()) &&
(PN->getParent() != L->getHeader() || !L->contains(BB))) {
// Split the critical edge.
BasicBlock *NewBB = SplitCriticalEdge(BB, PN->getParent(), P);
// If PN is outside of the loop and BB is in the loop, we want to
// move the block to be immediately before the PHI block, not
// immediately after BB.
if (L->contains(BB) && !L->contains(PN))
NewBB->moveBefore(PN->getParent());
// Splitting the edge can reduce the number of PHI entries we have.
e = PN->getNumIncomingValues();
BB = NewBB;
i = PN->getBasicBlockIndex(BB);
}
std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair =
Inserted.insert(std::make_pair(BB, static_cast<Value *>(0)));
if (!Pair.second)
PN->setIncomingValue(i, Pair.first->second);
else {
Value *FullV = Expand(BB->getTerminator(),
L, Rewriter, DeadInsts, SE, DT);
// If this is reuse-by-noop-cast, insert the noop cast.
if (FullV->getType() != OpTy)
FullV =
CastInst::Create(CastInst::getCastOpcode(FullV, false,
OpTy, false),
FullV, OperandValToReplace->getType(),
"tmp", BB->getTerminator());
PN->setIncomingValue(i, FullV);
Pair.first->second = FullV;
}
}
} else {
Value *FullV = Expand(UserInst, L, Rewriter, DeadInsts, SE, DT);
// If this is reuse-by-noop-cast, insert the noop cast.
if (FullV->getType() != OpTy) {
Instruction *Cast =
CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false),
FullV, OpTy, "tmp", UserInst);
FullV = Cast;
}
// Update the user.
UserInst->replaceUsesOfWith(OperandValToReplace, FullV);
}
DeadInsts.push_back(OperandValToReplace);
}
void LSRUse::print(raw_ostream &OS) const {
OS << "LSR Use: Kind=";
switch (Kind) {
case Basic: OS << "Basic"; break;
case Special: OS << "Special"; break;
case ICmpZero: OS << "ICmpZero"; break;
case Address:
OS << "Address of ";
if (isa<PointerType>(AccessTy))
OS << "pointer"; // the full pointer type could be really verbose
else
OS << *AccessTy;
}
OS << ", UserInst=";
// Store is common and interesting enough to be worth special-casing.
if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
OS << "store ";
WriteAsOperand(OS, Store->getOperand(0), /*PrintType=*/false);
} else if (UserInst->getType()->isVoidTy())
OS << UserInst->getOpcodeName();
else
WriteAsOperand(OS, UserInst, /*PrintType=*/false);
OS << ", OperandValToReplace=";
WriteAsOperand(OS, OperandValToReplace, /*PrintType=*/false);
if (PostIncLoop) {
OS << ", PostIncLoop=";
WriteAsOperand(OS, PostIncLoop->getHeader(), /*PrintType=*/false);
}
}
void LSRUse::dump() const {
print(errs()); errs() << '\n';
}
namespace {
/// Score - This class is used to measure and compare candidate formulae.
class Score {
unsigned NumRegs;
unsigned NumPhis;
unsigned NumIVMuls;
unsigned NumBaseAdds;
unsigned NumImms;
public:
Score()
: NumRegs(0), NumPhis(0), NumIVMuls(0), NumBaseAdds(0), NumImms(0) {}
void RateInitial(SmallVector<LSRUse, 16> const &Uses, const Loop *L,
ScalarEvolution &SE);
void Rate(const SCEV *Reg, const SmallBitVector &Bits,
const SmallVector<LSRUse, 16> &Uses, const Loop *L,
ScalarEvolution &SE);
bool operator<(const Score &Other) const;
void print_details(raw_ostream &OS, const SCEV *Reg,
const SmallPtrSet<const SCEV *, 8> &Regs) const;
void print(raw_ostream &OS) const;
void dump() const;
private:
void RateRegister(const SCEV *Reg, SmallPtrSet<const SCEV *, 8> &Regs,
const Loop *L);
void RateFormula(const Formula &F, SmallPtrSet<const SCEV *, 8> &Regs,
const Loop *L);
void Loose();
};
}
/// RateRegister - Tally up interesting quantities from the given register.
void Score::RateRegister(const SCEV *Reg,
SmallPtrSet<const SCEV *, 8> &Regs,
const Loop *L) {
if (Regs.insert(Reg))
if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) {
NumPhis += AR->getLoop() == L;
// Add the step value register, if it needs one.
if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1)))
RateRegister(AR->getOperand(1), Regs, L);
}
}
void Score::RateFormula(const Formula &F,
SmallPtrSet<const SCEV *, 8> &Regs,
const Loop *L) {
// Tally up the registers.
if (F.ScaledReg)
RateRegister(F.ScaledReg, Regs, L);
for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
E = F.BaseRegs.end(); I != E; ++I) {
const SCEV *BaseReg = *I;
RateRegister(BaseReg, Regs, L);
NumIVMuls += isa<SCEVMulExpr>(BaseReg) &&
BaseReg->hasComputableLoopEvolution(L);
}
if (F.BaseRegs.size() > 1)
NumBaseAdds += F.BaseRegs.size() - 1;
// Tally up the non-zero immediates.
if (F.AM.BaseGV || F.AM.BaseOffs != 0)
++NumImms;
}
/// Loose - Set this score to a loosing value.
void Score::Loose() {
NumRegs = ~0u;
NumPhis = ~0u;
NumIVMuls = ~0u;
NumBaseAdds = ~0u;
NumImms = ~0u;
}
/// RateInitial - Compute a score for the initial "fully reduced" solution.
void Score::RateInitial(SmallVector<LSRUse, 16> const &Uses, const Loop *L,
ScalarEvolution &SE) {
SmallPtrSet<const SCEV *, 8> Regs;
for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
E = Uses.end(); I != E; ++I)
RateFormula(I->Formulae.front(), Regs, L);
NumRegs += Regs.size();
DEBUG(print_details(dbgs(), 0, Regs));
}
/// Rate - Compute a score for the solution where the reuse associated with
/// putting Reg in a register is selected.
void Score::Rate(const SCEV *Reg, const SmallBitVector &Bits,
const SmallVector<LSRUse, 16> &Uses, const Loop *L,
ScalarEvolution &SE) {
SmallPtrSet<const SCEV *, 8> Regs;
for (size_t i = 0, e = Uses.size(); i != e; ++i) {
const LSRUse &LU = Uses[i];
const Formula *BestFormula = 0;
if (i >= Bits.size() || !Bits.test(i))
// This use doesn't use the current register. Just go with the current
// leading candidate formula.
BestFormula = &LU.Formulae.front();
else
// Find the best formula for this use that uses the current register.
for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(),
E = LU.Formulae.end(); I != E; ++I) {
const Formula &F = *I;
if (F.referencesReg(Reg) &&
(!BestFormula || ComplexitySorter()(F, *BestFormula)))
BestFormula = &F;
}
// If we didn't find *any* forumlae, because earlier we eliminated some
// in greedy fashion, skip the current register's reuse opportunity.
if (!BestFormula) {
DEBUG(dbgs() << "Reuse with reg " << *Reg
<< " wouldn't help any users.\n");
Loose();
return;
}
// For an in-loop post-inc user, don't allow multiple base registers,
// because that would require an awkward in-loop add after the increment.
if (LU.PostIncLoop && LU.PostIncLoop->contains(LU.UserInst) &&
BestFormula->BaseRegs.size() > 1) {
DEBUG(dbgs() << "Reuse with reg " << *Reg
<< " would require an in-loop post-inc add: ";
BestFormula->dump());
Loose();
return;
}
RateFormula(*BestFormula, Regs, L);
}
NumRegs += Regs.size();
DEBUG(print_details(dbgs(), Reg, Regs));
}
/// operator< - Choose the better score.
bool Score::operator<(const Score &Other) const {
if (NumRegs != Other.NumRegs)
return NumRegs < Other.NumRegs;
if (NumPhis != Other.NumPhis)
return NumPhis < Other.NumPhis;
if (NumIVMuls != Other.NumIVMuls)
return NumIVMuls < Other.NumIVMuls;
if (NumBaseAdds != Other.NumBaseAdds)
return NumBaseAdds < Other.NumBaseAdds;
return NumImms < Other.NumImms;
}
void Score::print_details(raw_ostream &OS,
const SCEV *Reg,
const SmallPtrSet<const SCEV *, 8> &Regs) const {
if (Reg) OS << "Reuse with reg " << *Reg << " would require ";
else OS << "The initial solution would require ";
print(OS);
OS << ". Regs:";
for (SmallPtrSet<const SCEV *, 8>::const_iterator I = Regs.begin(),
E = Regs.end(); I != E; ++I)
OS << ' ' << **I;
OS << '\n';
}
void Score::print(raw_ostream &OS) const {
OS << NumRegs << " reg" << (NumRegs == 1 ? "" : "s");
if (NumPhis != 0)
OS << ", including " << NumPhis << " PHI" << (NumPhis == 1 ? "" : "s");
if (NumIVMuls != 0)
OS << ", plus " << NumIVMuls << " IV mul" << (NumIVMuls == 1 ? "" : "s");
if (NumBaseAdds != 0)
OS << ", plus " << NumBaseAdds << " base add"
<< (NumBaseAdds == 1 ? "" : "s");
if (NumImms != 0)
OS << ", plus " << NumImms << " imm" << (NumImms == 1 ? "" : "s");
}
void Score::dump() const {
print(errs()); errs() << '\n';
}
/// isAddressUse - Returns true if the specified instruction is using the
/// specified value as an address.
static bool isAddressUse(Instruction *Inst, Value *OperandVal) {
bool isAddress = isa<LoadInst>(Inst);
if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
if (SI->getOperand(1) == OperandVal)
isAddress = true;
} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
// Addressing modes can also be folded into prefetches and a variety
// of intrinsics.
switch (II->getIntrinsicID()) {
default: break;
case Intrinsic::prefetch:
case Intrinsic::x86_sse2_loadu_dq:
case Intrinsic::x86_sse2_loadu_pd:
case Intrinsic::x86_sse_loadu_ps:
case Intrinsic::x86_sse_storeu_ps:
case Intrinsic::x86_sse2_storeu_pd:
case Intrinsic::x86_sse2_storeu_dq:
case Intrinsic::x86_sse2_storel_dq:
if (II->getOperand(1) == OperandVal)
isAddress = true;
break;
}
}
return isAddress;
}
/// getAccessType - Return the type of the memory being accessed.
static const Type *getAccessType(const Instruction *Inst) {
const Type *AccessTy = Inst->getType();
if (const StoreInst *SI = dyn_cast<StoreInst>(Inst))
AccessTy = SI->getOperand(0)->getType();
else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
// Addressing modes can also be folded into prefetches and a variety
// of intrinsics.
switch (II->getIntrinsicID()) {
default: break;
case Intrinsic::x86_sse_storeu_ps:
case Intrinsic::x86_sse2_storeu_pd:
case Intrinsic::x86_sse2_storeu_dq:
case Intrinsic::x86_sse2_storel_dq:
AccessTy = II->getOperand(1)->getType();
break;
}
}
return AccessTy;
}
/// DeleteTriviallyDeadInstructions - If any of the instructions is the
/// specified set are trivially dead, delete them and see if this makes any of
/// their operands subsequently dead.
static bool
DeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) {
bool Changed = false;
while (!DeadInsts.empty()) {
Instruction *I = dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val());
if (I == 0 || !isInstructionTriviallyDead(I))
continue;
for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI)
if (Instruction *U = dyn_cast<Instruction>(*OI)) {
*OI = 0;
if (U->use_empty())
DeadInsts.push_back(U);
}
I->eraseFromParent();
Changed = true;
}
return Changed;
}
namespace {
/// LSRInstance - This class holds state for the main loop strength
/// reduction logic.
class LSRInstance {
IVUsers &IU;
ScalarEvolution &SE;
DominatorTree &DT;
const TargetLowering *const TLI;
Loop *const L;
bool Changed;
/// CurrentArbitraryRegIndex - To ensure a deterministic ordering, assign an
/// arbitrary index value to each register as a sort tie breaker.
unsigned CurrentArbitraryRegIndex;
/// Factors - Interesting factors between use strides.
SmallSetVector<int64_t, 4> Factors;
/// Types - Interesting use types, to facilitate truncation reuse.
SmallSetVector<const Type *, 4> Types;
/// Uses - The list of interesting uses.
SmallVector<LSRUse, 16> Uses;
// TODO: Reorganize these data structures.
typedef DenseMap<const SCEV *, RegSortData> RegUsesTy;
RegUsesTy RegUses;
SmallVector<const SCEV *, 16> RegSequence;
void OptimizeShadowIV();
bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse,
const SCEV* &CondStride);
ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse);
bool StrideMightBeShared(const SCEV* Stride);
bool OptimizeLoopTermCond(Instruction *&IVIncInsertPos);
LSRUse &getNewUse() {
Uses.push_back(LSRUse());
return Uses.back();
}
void CountRegister(const SCEV *Reg, uint32_t Complexity, size_t LUIdx);
void CountRegisters(const Formula &F, size_t LUIdx);
void GenerateSymbolicOffsetReuse(LSRUse &LU, unsigned LUIdx,
Formula Base);
void GenerateICmpZeroScaledReuse(LSRUse &LU, unsigned LUIdx,
Formula Base);
void GenerateFormulaeFromReplacedBaseReg(LSRUse &LU,
unsigned LUIdx,
const Formula &Base, unsigned i,
const SmallVectorImpl<const SCEV *>
&AddOps);
void GenerateReassociationReuse(LSRUse &LU, unsigned LUIdx,
Formula Base);
void GenerateCombinationReuse(LSRUse &LU, unsigned LUIdx,
Formula Base);
void GenerateScaledReuse(LSRUse &LU, unsigned LUIdx,
Formula Base);
void GenerateTruncateReuse(LSRUse &LU, unsigned LUIdx,
Formula Base);
void GenerateConstantOffsetReuse();
void GenerateAllReuseFormulae();
void GenerateLoopInvariantRegisterUses();
public:
LSRInstance(const TargetLowering *tli, Loop *l, Pass *P);
bool getChanged() const { return Changed; }
void print(raw_ostream &OS) const;
void dump() const;
};
}
/// OptimizeShadowIV - If IV is used in a int-to-float cast
/// inside the loop then try to eliminate the cast opeation.
void LSRInstance::OptimizeShadowIV() {
const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
return;
for (size_t StrideIdx = 0, e = IU.StrideOrder.size();
StrideIdx != e; ++StrideIdx) {
std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI =
IU.IVUsesByStride.find(IU.StrideOrder[StrideIdx]);
assert(SI != IU.IVUsesByStride.end() && "Stride doesn't exist!");
if (!isa<SCEVConstant>(SI->first))
continue;
for (ilist<IVStrideUse>::iterator UI = SI->second->Users.begin(),
E = SI->second->Users.end(); UI != E; /* empty */) {
ilist<IVStrideUse>::iterator CandidateUI = UI;
++UI;
Instruction *ShadowUse = CandidateUI->getUser();
const Type *DestTy = NULL;
/* If shadow use is a int->float cast then insert a second IV
to eliminate this cast.
for (unsigned i = 0; i < n; ++i)
foo((double)i);
is transformed into
double d = 0.0;
for (unsigned i = 0; i < n; ++i, ++d)
foo(d);
*/
if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser()))
DestTy = UCast->getDestTy();
else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser()))
DestTy = SCast->getDestTy();
if (!DestTy) continue;
if (TLI) {
// If target does not support DestTy natively then do not apply
// this transformation.
EVT DVT = TLI->getValueType(DestTy);
if (!TLI->isTypeLegal(DVT)) continue;
}
PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0));
if (!PH) continue;
if (PH->getNumIncomingValues() != 2) continue;
const Type *SrcTy = PH->getType();
int Mantissa = DestTy->getFPMantissaWidth();
if (Mantissa == -1) continue;
if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa)
continue;
unsigned Entry, Latch;
if (PH->getIncomingBlock(0) == L->getLoopPreheader()) {
Entry = 0;
Latch = 1;
} else {
Entry = 1;
Latch = 0;
}
ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry));
if (!Init) continue;
Constant *NewInit = ConstantFP::get(DestTy, Init->getZExtValue());
BinaryOperator *Incr =
dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch));
if (!Incr) continue;
if (Incr->getOpcode() != Instruction::Add
&& Incr->getOpcode() != Instruction::Sub)
continue;
/* Initialize new IV, double d = 0.0 in above example. */
ConstantInt *C = NULL;
if (Incr->getOperand(0) == PH)
C = dyn_cast<ConstantInt>(Incr->getOperand(1));
else if (Incr->getOperand(1) == PH)
C = dyn_cast<ConstantInt>(Incr->getOperand(0));
else
continue;
if (!C) continue;
// Ignore negative constants, as the code below doesn't handle them
// correctly. TODO: Remove this restriction.
if (!C->getValue().isStrictlyPositive()) continue;
/* Add new PHINode. */
PHINode *NewPH = PHINode::Create(DestTy, "IV.S.", PH);
/* create new increment. '++d' in above example. */
Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue());
BinaryOperator *NewIncr =
BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ?
Instruction::FAdd : Instruction::FSub,
NewPH, CFP, "IV.S.next.", Incr);
NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry));
NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch));
/* Remove cast operation */
ShadowUse->replaceAllUsesWith(NewPH);
ShadowUse->eraseFromParent();
break;
}
}
}
/// FindIVUserForCond - If Cond has an operand that is an expression of an IV,
/// set the IV user and stride information and return true, otherwise return
/// false.
bool LSRInstance::FindIVUserForCond(ICmpInst *Cond,
IVStrideUse *&CondUse,
const SCEV* &CondStride) {
for (unsigned StrideIdx = 0, e = IU.StrideOrder.size();
StrideIdx != e && !CondUse; ++StrideIdx) {
std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI =
IU.IVUsesByStride.find(IU.StrideOrder[StrideIdx]);
assert(SI != IU.IVUsesByStride.end() && "Stride doesn't exist!");
for (ilist<IVStrideUse>::iterator UI = SI->second->Users.begin(),
E = SI->second->Users.end(); UI != E; ++UI)
if (UI->getUser() == Cond) {
// NOTE: we could handle setcc instructions with multiple uses here, but
// InstCombine does it as well for simple uses, it's not clear that it
// occurs enough in real life to handle.
CondUse = UI;
CondStride = SI->first;
return true;
}
}
return false;
}
/// OptimizeMax - Rewrite the loop's terminating condition if it uses
/// a max computation.
///
/// This is a narrow solution to a specific, but acute, problem. For loops
/// like this:
///
/// i = 0;
/// do {
/// p[i] = 0.0;
/// } while (++i < n);
///
/// the trip count isn't just 'n', because 'n' might not be positive. And
/// unfortunately this can come up even for loops where the user didn't use
/// a C do-while loop. For example, seemingly well-behaved top-test loops
/// will commonly be lowered like this:
//
/// if (n > 0) {
/// i = 0;
/// do {
/// p[i] = 0.0;
/// } while (++i < n);
/// }
///
/// and then it's possible for subsequent optimization to obscure the if
/// test in such a way that indvars can't find it.
///
/// When indvars can't find the if test in loops like this, it creates a
/// max expression, which allows it to give the loop a canonical
/// induction variable:
///
/// i = 0;
/// max = n < 1 ? 1 : n;
/// do {
/// p[i] = 0.0;
/// } while (++i != max);
///
/// Canonical induction variables are necessary because the loop passes
/// are designed around them. The most obvious example of this is the
/// LoopInfo analysis, which doesn't remember trip count values. It
/// expects to be able to rediscover the trip count each time it is
/// needed, and it does this using a simple analysis that only succeeds if
/// the loop has a canonical induction variable.
///
/// However, when it comes time to generate code, the maximum operation
/// can be quite costly, especially if it's inside of an outer loop.
///
/// This function solves this problem by detecting this type of loop and
/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
/// the instructions for the maximum computation.
///
ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) {
// Check that the loop matches the pattern we're looking for.
if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
Cond->getPredicate() != CmpInst::ICMP_NE)
return Cond;
SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1));
if (!Sel || !Sel->hasOneUse()) return Cond;
const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
return Cond;
const SCEV *One = SE.getIntegerSCEV(1, BackedgeTakenCount->getType());
// Add one to the backedge-taken count to get the trip count.
const SCEV *IterationCount = SE.getAddExpr(BackedgeTakenCount, One);
// Check for a max calculation that matches the pattern.
if (!isa<SCEVSMaxExpr>(IterationCount) && !isa<SCEVUMaxExpr>(IterationCount))
return Cond;
const SCEVNAryExpr *Max = cast<SCEVNAryExpr>(IterationCount);
if (Max != SE.getSCEV(Sel)) return Cond;
// To handle a max with more than two operands, this optimization would
// require additional checking and setup.
if (Max->getNumOperands() != 2)
return Cond;
const SCEV *MaxLHS = Max->getOperand(0);
const SCEV *MaxRHS = Max->getOperand(1);
if (!MaxLHS || MaxLHS != One) return Cond;
// Check the relevant induction variable for conformance to
// the pattern.
const SCEV *IV = SE.getSCEV(Cond->getOperand(0));
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV);
if (!AR || !AR->isAffine() ||
AR->getStart() != One ||
AR->getStepRecurrence(SE) != One)
return Cond;
assert(AR->getLoop() == L &&
"Loop condition operand is an addrec in a different loop!");
// Check the right operand of the select, and remember it, as it will
// be used in the new comparison instruction.
Value *NewRHS = 0;
if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS)
NewRHS = Sel->getOperand(1);
else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS)
NewRHS = Sel->getOperand(2);
if (!NewRHS) return Cond;
// Determine the new comparison opcode. It may be signed or unsigned,
// and the original comparison may be either equality or inequality.
CmpInst::Predicate Pred =
isa<SCEVSMaxExpr>(Max) ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT;
if (Cond->getPredicate() == CmpInst::ICMP_EQ)
Pred = CmpInst::getInversePredicate(Pred);
// Ok, everything looks ok to change the condition into an SLT or SGE and
// delete the max calculation.
ICmpInst *NewCond =
new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp");
// Delete the max calculation instructions.
Cond->replaceAllUsesWith(NewCond);
CondUse->setUser(NewCond);
Instruction *Cmp = cast<Instruction>(Sel->getOperand(0));
Cond->eraseFromParent();
Sel->eraseFromParent();
if (Cmp->use_empty())
Cmp->eraseFromParent();
return NewCond;
}
bool LSRInstance::StrideMightBeShared(const SCEV* Stride) {
int64_t SInt = cast<SCEVConstant>(Stride)->getValue()->getSExtValue();
for (unsigned i = 0, e = IU.StrideOrder.size(); i != e; ++i) {
std::map<const SCEV *, IVUsersOfOneStride *>::iterator SI =
IU.IVUsesByStride.find(IU.StrideOrder[i]);
const SCEV *Share = SI->first;
if (!isa<SCEVConstant>(SI->first) || Share == Stride)
continue;
int64_t SSInt = cast<SCEVConstant>(Share)->getValue()->getSExtValue();
if (SSInt == SInt)
return true; // This can definitely be reused.
if (unsigned(abs64(SSInt)) < SInt || (SSInt % SInt) != 0)
continue;
int64_t Scale = SSInt / SInt;
// This AM will be used for conservative queries. At this point in the
// process we don't know which users will have a base reg, immediate,
// etc., so we conservatively assume that it may not, making more
// strides valid, thus erring on the side of assuming that there
// might be sharing.
TargetLowering::AddrMode AM;
AM.Scale = Scale;
// Any pre-inc iv use?
IVUsersOfOneStride &StrideUses = *IU.IVUsesByStride[Share];
for (ilist<IVStrideUse>::iterator I = StrideUses.Users.begin(),
E = StrideUses.Users.end(); I != E; ++I) {
bool isAddress = isAddressUse(I->getUser(), I->getOperandValToReplace());
if (!I->isUseOfPostIncrementedValue() &&
isLegalUse(AM, isAddress ? LSRUse::Address : LSRUse::Basic,
isAddress ? getAccessType(I->getUser()) : 0,
TLI))
return true;
}
}
return false;
}
/// OptimizeLoopTermCond - Change loop terminating condition to use the
/// postinc iv when possible.
bool
LSRInstance::OptimizeLoopTermCond(Instruction *&IVIncInsertPos) {
SmallPtrSet<Instruction *, 4> PostIncs;
BasicBlock *LatchBlock = L->getLoopLatch();
SmallVector<BasicBlock*, 8> ExitingBlocks;
L->getExitingBlocks(ExitingBlocks);
for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
BasicBlock *ExitingBlock = ExitingBlocks[i];
// Get the terminating condition for the loop if possible. If we
// can, we want to change it to use a post-incremented version of its
// induction variable, to allow coalescing the live ranges for the IV into
// one register value.
BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator());
if (!TermBr)
continue;
// FIXME: Overly conservative, termination condition could be an 'or' etc..
if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition()))
continue;
// Search IVUsesByStride to find Cond's IVUse if there is one.
IVStrideUse *CondUse = 0;
const SCEV *CondStride = 0;
ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition());
if (!FindIVUserForCond(Cond, CondUse, CondStride))
continue;
// If the trip count is computed in terms of a max (due to ScalarEvolution
// being unable to find a sufficient guard, for example), change the loop
// comparison to use SLT or ULT instead of NE.
// One consequence of doing this now is that it disrupts the count-down
// optimization. That's not always a bad thing though, because in such
// cases it may still be worthwhile to avoid a max.
Cond = OptimizeMax(Cond, CondUse);
// If this exiting block is the latch block, and the condition has only
// one use inside the loop (the branch), use the post-incremented value
// of the induction variable
if (ExitingBlock != LatchBlock) {
// If this exiting block dominates the latch block, it may also use
// the post-inc value if it won't be shared with other uses.
// Check for dominance.
if (!DT.dominates(ExitingBlock, LatchBlock))
continue;
// Check for sharing within the same stride.
bool SameStrideSharing = false;
IVUsersOfOneStride &StrideUses = *IU.IVUsesByStride[CondStride];
for (ilist<IVStrideUse>::iterator I = StrideUses.Users.begin(),
E = StrideUses.Users.end(); I != E; ++I) {
if (I->getUser() == Cond)
continue;
if (!I->isUseOfPostIncrementedValue()) {
SameStrideSharing = true;
break;
}
}
if (SameStrideSharing)
continue;
// Check for sharing from a different stride.
if (isa<SCEVConstant>(CondStride) && StrideMightBeShared(CondStride))
continue;
}
if (!Cond->hasOneUse()) {
bool HasOneUseInLoop = true;
for (Value::use_iterator UI = Cond->use_begin(), UE = Cond->use_end();
UI != UE; ++UI) {
Instruction *U = cast<Instruction>(*UI);
if (U == TermBr)
continue;
if (L->contains(U)) {
HasOneUseInLoop = false;
break;
}
}
if (!HasOneUseInLoop)
continue;
}
DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "
<< *Cond << '\n');
// It's possible for the setcc instruction to be anywhere in the loop, and
// possible for it to have multiple users. If it is not immediately before
// the exiting block branch, move it.
if (&*++BasicBlock::iterator(Cond) != TermBr) {
if (Cond->hasOneUse()) { // Condition has a single use, just move it.
Cond->moveBefore(TermBr);
} else {
// Otherwise, clone the terminating condition and insert into the
// loopend.
ICmpInst *OldCond = Cond;
Cond = cast<ICmpInst>(Cond->clone());
Cond->setName(L->getHeader()->getName() + ".termcond");
ExitingBlock->getInstList().insert(TermBr, Cond);
// Clone the IVUse, as the old use still exists!
IU.IVUsesByStride[CondStride]->addUser(CondUse->getOffset(), Cond,
CondUse->getOperandValToReplace());
CondUse = &IU.IVUsesByStride[CondStride]->Users.back();
TermBr->replaceUsesOfWith(OldCond, Cond);
}
}
// If we get to here, we know that we can transform the setcc instruction to
// use the post-incremented version of the IV, allowing us to coalesce the
// live ranges for the IV correctly.
CondUse->setOffset(SE.getMinusSCEV(CondUse->getOffset(), CondStride));
CondUse->setIsUseOfPostIncrementedValue(true);
Changed = true;
PostIncs.insert(Cond);
}
// Determine an insertion point for the loop induction variable increment. It
// must dominate all the post-inc comparisons we just set up, and it must
// dominate the loop latch edge.
IVIncInsertPos = L->getLoopLatch()->getTerminator();
for (SmallPtrSet<Instruction *, 4>::iterator I = PostIncs.begin(),
E = PostIncs.end(); I != E; ++I) {
BasicBlock *BB =
DT.findNearestCommonDominator(IVIncInsertPos->getParent(),
(*I)->getParent());
if (BB == (*I)->getParent())
IVIncInsertPos = *I;
else if (BB != IVIncInsertPos->getParent())
IVIncInsertPos = BB->getTerminator();
}
return Changed;
}
/// CountRegisters - Note the given register.
void LSRInstance::CountRegister(const SCEV *Reg, uint32_t Complexity,
size_t LUIdx) {
std::pair<RegUsesTy::iterator, bool> Pair =
RegUses.insert(std::make_pair(Reg, RegSortData()));
RegSortData &BV = Pair.first->second;
if (Pair.second) {
BV.Index = CurrentArbitraryRegIndex++;
BV.MaxComplexity = Complexity;
RegSequence.push_back(Reg);
} else {
BV.MaxComplexity = std::max(BV.MaxComplexity, Complexity);
}
BV.Bits.resize(std::max(BV.Bits.size(), LUIdx + 1));
BV.Bits.set(LUIdx);
}
/// CountRegisters - Note which registers are used by the given formula,
/// updating RegUses.
void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
uint32_t Complexity = F.getComplexity();
if (F.ScaledReg)
CountRegister(F.ScaledReg, Complexity, LUIdx);
for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
E = F.BaseRegs.end(); I != E; ++I)
CountRegister(*I, Complexity, LUIdx);
}
/// GenerateSymbolicOffsetReuse - Generate reuse formulae using symbolic
/// offsets.
void LSRInstance::GenerateSymbolicOffsetReuse(LSRUse &LU, unsigned LUIdx,
Formula Base) {
// We can't add a symbolic offset if the address already contains one.
if (Base.AM.BaseGV) return;
for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) {
const SCEV *G = Base.BaseRegs[i];
GlobalValue *GV = ExtractSymbol(G, SE);
if (G->isZero())
continue;
Formula F = Base;
F.AM.BaseGV = GV;
if (!isLegalUse(F.AM, LU.Kind, LU.AccessTy, TLI))
continue;
F.BaseRegs[i] = G;
if (LU.InsertFormula(F))
CountRegisters(LU.Formulae.back(), LUIdx);
}
}
/// GenerateICmpZeroScaledReuse - For ICmpZero, check to see if we can scale up
/// the comparison. For example, x == y -> x*c == y*c.
void LSRInstance::GenerateICmpZeroScaledReuse(LSRUse &LU, unsigned LUIdx,
Formula Base) {
if (LU.Kind != LSRUse::ICmpZero) return;
// Determine the integer type for the base formula.
const Type *IntTy = Base.getType();
if (!IntTy) return;
if (SE.getTypeSizeInBits(IntTy) > 64) return;
IntTy = SE.getEffectiveSCEVType(IntTy);
assert(!Base.AM.BaseGV && "ICmpZero use is not legal!");
// Check each interesting stride.
for (SmallSetVector<int64_t, 4>::const_iterator
I = Factors.begin(), E = Factors.end(); I != E; ++I) {
int64_t Factor = *I;
Formula F = Base;
// Check that the multiplication doesn't overflow.
F.AM.BaseOffs = (uint64_t)F.AM.BaseOffs * Factor;
if ((int64_t)F.AM.BaseOffs / Factor != F.AM.BaseOffs)
continue;
// Check that this scale is legal.
if (!isLegalUse(F.AM, LU.Kind, LU.AccessTy, TLI))
continue;
const SCEV *FactorS = SE.getSCEV(ConstantInt::get(IntTy, Factor));
// Check that multiplying with each base register doesn't overflow.
for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) {
F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS);
if (getSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i])
goto next;
}
// Check that multiplying with the scaled register doesn't overflow.
if (F.ScaledReg) {
F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS);
if (getSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg)
continue;
}
// If we make it here and it's legal, add it.
if (LU.InsertFormula(F))
CountRegisters(LU.Formulae.back(), LUIdx);
next:;
}
}
/// GenerateFormulaeFromReplacedBaseReg - If removing base register with
/// index i from the BaseRegs list and adding the registers in AddOps
/// to the address forms an interesting formula, pursue it.
void
LSRInstance::GenerateFormulaeFromReplacedBaseReg(
LSRUse &LU,
unsigned LUIdx,
const Formula &Base, unsigned i,
const SmallVectorImpl<const SCEV *>
&AddOps) {
if (AddOps.empty()) return;
Formula F = Base;
std::swap(F.BaseRegs[i], F.BaseRegs.back());
F.BaseRegs.pop_back();
for (SmallVectorImpl<const SCEV *>::const_iterator I = AddOps.begin(),
E = AddOps.end(); I != E; ++I)
F.BaseRegs.push_back(*I);
F.AM.HasBaseReg = !F.BaseRegs.empty();
if (LU.InsertFormula(F)) {
CountRegisters(LU.Formulae.back(), LUIdx);
// Recurse.
GenerateReassociationReuse(LU, LUIdx, LU.Formulae.back());
}
}
/// GenerateReassociationReuse - Split out subexpressions from adds and
/// the bases of addrecs.
void LSRInstance::GenerateReassociationReuse(LSRUse &LU, unsigned LUIdx,
Formula Base) {
SmallVector<const SCEV *, 8> AddOps;
for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) {
const SCEV *BaseReg = Base.BaseRegs[i];
if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BaseReg)) {
for (SCEVAddExpr::op_iterator J = Add->op_begin(), JE = Add->op_end();
J != JE; ++J) {
// Don't pull a constant into a register if the constant could be
// folded into an immediate field.
if (isAlwaysFoldable(*J, true, LU.Kind, LU.AccessTy, TLI, SE)) continue;
SmallVector<const SCEV *, 8> InnerAddOps;
for (SCEVAddExpr::op_iterator K = Add->op_begin(); K != JE; ++K)
if (K != J)
InnerAddOps.push_back(*K);
// Splitting a 2-operand add both ways is redundant. Pruning this
// now saves compile time.
if (InnerAddOps.size() < 2 && next(J) == JE)
continue;
AddOps.push_back(*J);
const SCEV *InnerAdd = SE.getAddExpr(InnerAddOps);
AddOps.push_back(InnerAdd);
GenerateFormulaeFromReplacedBaseReg(LU, LUIdx, Base, i, AddOps);
AddOps.clear();
}
} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(BaseReg)) {
if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(AR->getStart())) {
for (SCEVAddExpr::op_iterator J = Add->op_begin(), JE = Add->op_end();
J != JE; ++J) {
// Don't pull a constant into a register if the constant could be
// folded into an immediate field.
if (isAlwaysFoldable(*J, true, LU.Kind, LU.AccessTy, TLI, SE))
continue;
SmallVector<const SCEV *, 8> InnerAddOps;
for (SCEVAddExpr::op_iterator K = Add->op_begin(); K != JE; ++K)
if (K != J)
InnerAddOps.push_back(*K);
AddOps.push_back(*J);
const SCEV *InnerAdd = SE.getAddExpr(InnerAddOps);
AddOps.push_back(SE.getAddRecExpr(InnerAdd,
AR->getStepRecurrence(SE),
AR->getLoop()));
GenerateFormulaeFromReplacedBaseReg(LU, LUIdx, Base, i, AddOps);
AddOps.clear();
}
} else if (!isAlwaysFoldable(AR->getStart(), Base.BaseRegs.size() > 1,
LU.Kind, LU.AccessTy,
TLI, SE)) {
AddOps.push_back(AR->getStart());
AddOps.push_back(SE.getAddRecExpr(SE.getIntegerSCEV(0,
BaseReg->getType()),
AR->getStepRecurrence(SE),
AR->getLoop()));
GenerateFormulaeFromReplacedBaseReg(LU, LUIdx, Base, i, AddOps);
AddOps.clear();
}
}
}
}
/// GenerateCombinationReuse - Generate a formula consisting of all of the
/// loop-dominating registers added into a single register.
void LSRInstance::GenerateCombinationReuse(LSRUse &LU, unsigned LUIdx,
Formula Base) {
if (Base.BaseRegs.size() <= 1) return;
Formula F = Base;
F.BaseRegs.clear();
SmallVector<const SCEV *, 4> Ops;
for (SmallVectorImpl<const SCEV *>::const_iterator
I = Base.BaseRegs.begin(), E = Base.BaseRegs.end(); I != E; ++I) {
const SCEV *BaseReg = *I;
if (BaseReg->properlyDominates(L->getHeader(), &DT) &&
!BaseReg->hasComputableLoopEvolution(L))
Ops.push_back(BaseReg);
else
F.BaseRegs.push_back(BaseReg);
}
if (Ops.size() > 1) {
F.BaseRegs.push_back(SE.getAddExpr(Ops));
if (LU.InsertFormula(F))
CountRegisters(LU.Formulae.back(), LUIdx);
}
}
/// GenerateScaledReuse - Generate stride factor reuse formulae by making
/// use of scaled-offset address modes, for example.
void LSRInstance::GenerateScaledReuse(LSRUse &LU, unsigned LUIdx,
Formula Base) {
// Determine the integer type for the base formula.
const Type *IntTy = Base.getType();
if (!IntTy) return;
IntTy = SE.getEffectiveSCEVType(IntTy);
// Check each interesting stride.
for (SmallSetVector<int64_t, 4>::const_iterator
I = Factors.begin(), E = Factors.end(); I != E; ++I) {
int64_t Factor = *I;
// If this Formula already has a scaled register, we can't add another one.
if (Base.AM.Scale != 0)
continue;
Formula F = Base;
F.AM.Scale = Factor;
// Check whether this scale is going to be legal.
if (!isLegalUse(F.AM, LU.Kind, LU.AccessTy, TLI)) {
// As a special-case, handle special out-of-loop Basic users specially.
// TODO: Reconsider this special case.
if (LU.Kind == LSRUse::Basic &&
isLegalUse(F.AM, LSRUse::Special, LU.AccessTy, TLI) &&
!L->contains(LU.UserInst))
LU.Kind = LSRUse::Special;
else
continue;
}
// For each addrec base reg, apply the scale, if possible.
for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
if (const SCEVAddRecExpr *AR =
dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i])) {
const SCEV *FactorS = SE.getSCEV(ConstantInt::get(IntTy, Factor));
// Divide out the factor, ignoring high bits, since we'll be
// scaling the value back up in the end.
if (const SCEV *Quotient = getSDiv(AR, FactorS, SE, true)) {
// TODO: This could be optimized to avoid all the copying.
Formula NewF = F;
NewF.ScaledReg = Quotient;
std::swap(NewF.BaseRegs[i], NewF.BaseRegs.back());
NewF.BaseRegs.pop_back();
NewF.AM.HasBaseReg = !NewF.BaseRegs.empty();
if (LU.InsertFormula(NewF))
CountRegisters(LU.Formulae.back(), LUIdx);
}
}
}
}
/// GenerateTruncateReuse - Generate reuse formulae from different IV types.
void LSRInstance::GenerateTruncateReuse(LSRUse &LU, unsigned LUIdx,
Formula Base) {
// This requires TargetLowering to tell us which truncates are free.
if (!TLI) return;
// Don't attempt to truncate symbolic values.
if (Base.AM.BaseGV) return;
// Determine the integer type for the base formula.
const Type *DstTy = Base.getType();
if (!DstTy) return;
DstTy = SE.getEffectiveSCEVType(DstTy);
for (SmallSetVector<const Type *, 4>::const_iterator
I = Types.begin(), E = Types.end(); I != E; ++I) {
const Type *SrcTy = *I;
if (SrcTy != DstTy && TLI->isTruncateFree(SrcTy, DstTy)) {
Formula F = Base;
if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, *I);
for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(),
JE = F.BaseRegs.end(); J != JE; ++J)
*J = SE.getAnyExtendExpr(*J, SrcTy);
if (LU.InsertFormula(F))
CountRegisters(LU.Formulae.back(), LUIdx);
}
}
}
namespace {
/// WorkItem - Helper class for GenerateConstantOffsetReuse. It's used to
/// defer modifications so that the search phase doesn't have to worry about
/// the data structures moving underneath it.
struct WorkItem {
LSRUse *LU;
size_t LUIdx;
int64_t Imm;
const SCEV *OrigReg;
WorkItem(LSRUse *U, size_t LI, int64_t I, const SCEV *R)
: LU(U), LUIdx(LI), Imm(I), OrigReg(R) {}
void print(raw_ostream &OS) const;
void dump() const;
};
void WorkItem::print(raw_ostream &OS) const {
OS << "in use ";
LU->print(OS);
OS << " (at index " << LUIdx << "), add offset " << Imm
<< " and compensate by adjusting refences to " << *OrigReg << "\n";
}
void WorkItem::dump() const {
print(errs()); errs() << '\n';
}
}
/// GenerateConstantOffsetReuse - Look for registers which are a constant
/// distance apart and try to form reuse opportunities between them.
void LSRInstance::GenerateConstantOffsetReuse() {
// Group the registers by their value without any added constant offset.
typedef std::map<int64_t, const SCEV *> ImmMapTy;
typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy;
RegMapTy Map;
SmallVector<const SCEV *, 8> Sequence;
for (SmallVectorImpl<const SCEV *>::iterator I = RegSequence.begin(),
E = RegSequence.end(); I != E; ++I) {
const SCEV *Reg = *I;
int64_t Imm = ExtractImmediate(Reg, SE);
std::pair<RegMapTy::iterator, bool> Pair =
Map.insert(std::make_pair(Reg, ImmMapTy()));
if (Pair.second)
Sequence.push_back(Reg);
Pair.first->second.insert(std::make_pair(Imm, *I));
}
// Insert an artificial expression at offset 0 (if there isn't one already),
// as this may lead to more reuse opportunities.
for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(),
E = Sequence.end(); I != E; ++I)
Map.find(*I)->second.insert(ImmMapTy::value_type(0, 0));
// Now examine each set of registers with the same base value. Build up
// a list of work to do and do the work in a separate step so that we're
// not adding formulae and register counts while we're searching.
SmallVector<WorkItem, 32> WorkItems;
for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(),
E = Sequence.end(); I != E; ++I) {
const SCEV *Reg = *I;
const ImmMapTy &Imms = Map.find(Reg)->second;
// Examine each offset.
for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
J != JE; ++J) {
const SCEV *OrigReg = J->second;
// Skip the artifical register at offset 0.
if (!OrigReg) continue;
int64_t JImm = J->first;
const SmallBitVector &Bits = RegUses.find(OrigReg)->second.Bits;
// Examine each other offset associated with the same register. This is
// quadradic in the number of registers with the same base, but it's
// uncommon for this to be a large number.
for (ImmMapTy::const_iterator M = Imms.begin(); M != JE; ++M) {
if (M == J) continue;
// Compute the difference between the two.
int64_t Imm = (uint64_t)JImm - M->first;
for (int LUIdx = Bits.find_first(); LUIdx != -1;
LUIdx = Bits.find_next(LUIdx))
// Make a memo of this use, offset, and register tuple.
WorkItems.push_back(WorkItem(&Uses[LUIdx], LUIdx, Imm, OrigReg));
}
}
}
// Now iterate through the worklist and add new formulae.
for (SmallVectorImpl<WorkItem>::const_iterator I = WorkItems.begin(),
E = WorkItems.end(); I != E; ++I) {
const WorkItem &WI = *I;
LSRUse &LU = *WI.LU;
size_t LUIdx = WI.LUIdx;
int64_t Imm = WI.Imm;
const SCEV *OrigReg = WI.OrigReg;
const Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType());
const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy,
-(uint64_t)Imm));
for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
Formula F = LU.Formulae[L];
// Use the immediate in the scaled register.
if (F.ScaledReg == OrigReg) {
int64_t Offs = (uint64_t)F.AM.BaseOffs +
Imm * (uint64_t)F.AM.Scale;
// Don't create 50 + reg(-50).
if (F.referencesReg(SE.getSCEV(
ConstantInt::get(IntTy, -(uint64_t)Offs))))
continue;
Formula NewF = F;
NewF.AM.BaseOffs = Offs;
if (!isLegalUse(NewF.AM, LU.Kind, LU.AccessTy, TLI))
continue;
const SCEV *Diff = SE.getAddExpr(NegImmS, NewF.ScaledReg);
if (Diff->isZero()) continue;
NewF.ScaledReg = Diff;
if (LU.InsertFormula(NewF))
CountRegisters(LU.Formulae.back(), LUIdx);
}
// Use the immediate in a base register.
for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) {
const SCEV *BaseReg = F.BaseRegs[N];
if (BaseReg != OrigReg)
continue;
Formula NewF = F;
NewF.AM.BaseOffs = (uint64_t)NewF.AM.BaseOffs + Imm;
if (!isLegalUse(NewF.AM, LU.Kind, LU.AccessTy, TLI))
continue;
const SCEV *Diff = SE.getAddExpr(NegImmS, BaseReg);
if (Diff->isZero()) continue;
// Don't create 50 + reg(-50).
if (Diff ==
SE.getSCEV(ConstantInt::get(IntTy,
-(uint64_t)NewF.AM.BaseOffs)))
continue;
NewF.BaseRegs[N] = Diff;
if (LU.InsertFormula(NewF))
CountRegisters(LU.Formulae.back(), LUIdx);
}
}
}
}
/// GenerateAllReuseFormulae - Generate formulae for each use.
void
LSRInstance::GenerateAllReuseFormulae() {
SmallVector<Formula, 12> Save;
for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
LSRUse &LU = Uses[LUIdx];
for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
GenerateSymbolicOffsetReuse(LU, LUIdx, LU.Formulae[i]);
for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
GenerateICmpZeroScaledReuse(LU, LUIdx, LU.Formulae[i]);
for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
GenerateReassociationReuse(LU, LUIdx, LU.Formulae[i]);
for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
GenerateCombinationReuse(LU, LUIdx, LU.Formulae[i]);
for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
GenerateScaledReuse(LU, LUIdx, LU.Formulae[i]);
for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
GenerateTruncateReuse(LU, LUIdx, LU.Formulae[i]);
}
GenerateConstantOffsetReuse();
}
/// GenerateLoopInvariantRegisterUses - Check for other uses of loop-invariant
/// values which we're tracking. These other uses will pin these values in
/// registers, making them less profitable for elimination.
/// TODO: This currently misses non-constant addrec step registers.
/// TODO: Should this give more weight to users inside the loop?
void
LSRInstance::GenerateLoopInvariantRegisterUses() {
for (size_t i = 0, e = RegSequence.size(); i != e; ++i)
if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(RegSequence[i])) {
const Value *V = U->getValue();
if (const Instruction *Inst = dyn_cast<Instruction>(V))
if (L->contains(Inst)) continue;
for (Value::use_const_iterator UI = V->use_begin(), UE = V->use_end();
UI != UE; ++UI) {
const Instruction *UserInst = dyn_cast<Instruction>(*UI);
// Ignore non-instructions.
if (!UserInst)
continue;
// Ignore instructions in other functions (as can happen with
// Constants).
if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
continue;
// Ignore instructions not dominated by the loop.
const BasicBlock *UseBB = !isa<PHINode>(UserInst) ?
UserInst->getParent() :
cast<PHINode>(UserInst)->getIncomingBlock(
PHINode::getIncomingValueNumForOperand(UI.getOperandNo()));
if (!DT.dominates(L->getHeader(), UseBB))
continue;
// Ignore uses which are part of other SCEV expressions, to avoid
// analyzing them multiple times.
if (SE.isSCEVable(UserInst->getType()) &&
!isa<SCEVUnknown>(SE.getSCEV(const_cast<Instruction *>(UserInst))))
continue;
// Ignore icmp instructions which are already being analyzed.
if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) {
unsigned OtherIdx = !UI.getOperandNo();
Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx));
if (SE.getSCEV(OtherOp)->hasComputableLoopEvolution(L))
continue;
}
LSRUse &LU = getNewUse();
LU.UserInst = const_cast<Instruction *>(UserInst);
LU.OperandValToReplace = UI.getUse();
LU.InsertSupplementalFormula(U);
CountRegisters(LU.Formulae.back(), Uses.size() - 1);
}
}
}
#ifndef NDEBUG
static void debug_winner(SmallVector<LSRUse, 16> const &Uses) {
dbgs() << "LSR has selected formulae for each use:\n";
for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
E = Uses.end(); I != E; ++I) {
const LSRUse &LU = *I;
dbgs() << " ";
LU.print(dbgs());
dbgs() << '\n';
dbgs() << " ";
LU.Formulae.front().print(dbgs());
dbgs() << "\n";
}
}
#endif
LSRInstance::LSRInstance(const TargetLowering *tli, Loop *l, Pass *P)
: IU(P->getAnalysis<IVUsers>()),
SE(P->getAnalysis<ScalarEvolution>()),
DT(P->getAnalysis<DominatorTree>()),
TLI(tli), L(l), Changed(false), CurrentArbitraryRegIndex(0) {
// If LoopSimplify form is not available, stay out of trouble.
if (!L->isLoopSimplifyForm()) return;
// If there's no interesting work to be done, bail early.
if (IU.IVUsesByStride.empty()) return;
DEBUG(dbgs() << "\nLSR on loop ";
WriteAsOperand(dbgs(), L->getHeader(), /*PrintType=*/false);
dbgs() << ":\n");
// Sort the StrideOrder so we process larger strides first.
std::stable_sort(IU.StrideOrder.begin(), IU.StrideOrder.end(),
StrideCompare(SE));
/// OptimizeShadowIV - If IV is used in a int-to-float cast
/// inside the loop then try to eliminate the cast opeation.
OptimizeShadowIV();
// Change loop terminating condition to use the postinc iv when possible.
Instruction *IVIncInsertPos;
Changed |= OptimizeLoopTermCond(IVIncInsertPos);
for (SmallVectorImpl<const SCEV *>::const_iterator SIter =
IU.StrideOrder.begin(), SEnd = IU.StrideOrder.end();
SIter != SEnd; ++SIter) {
const SCEV *Stride = *SIter;
// Collect interesting types.
Types.insert(SE.getEffectiveSCEVType(Stride->getType()));
// Collect interesting factors.
for (SmallVectorImpl<const SCEV *>::const_iterator NewStrideIter =
SIter + 1; NewStrideIter != SEnd; ++NewStrideIter) {
const SCEV *OldStride = Stride;
const SCEV *NewStride = *NewStrideIter;
if (SE.getTypeSizeInBits(OldStride->getType()) !=
SE.getTypeSizeInBits(NewStride->getType())) {
if (SE.getTypeSizeInBits(OldStride->getType()) >
SE.getTypeSizeInBits(NewStride->getType()))
NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType());
else
OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType());
}
if (const SCEVConstant *Factor =
dyn_cast_or_null<SCEVConstant>(getSDiv(NewStride, OldStride,
SE, true)))
if (Factor->getValue()->getValue().getMinSignedBits() <= 64)
Factors.insert(Factor->getValue()->getValue().getSExtValue());
}
std::map<const SCEV *, IVUsersOfOneStride *>::const_iterator SI =
IU.IVUsesByStride.find(Stride);
assert(SI != IU.IVUsesByStride.end() && "Stride doesn't exist!");
for (ilist<IVStrideUse>::const_iterator UI = SI->second->Users.begin(),
E = SI->second->Users.end(); UI != E; ++UI) {
// Record the uses.
LSRUse &LU = getNewUse();
LU.UserInst = UI->getUser();
LU.OperandValToReplace = UI->getOperandValToReplace();
if (isAddressUse(LU.UserInst, LU.OperandValToReplace)) {
LU.Kind = LSRUse::Address;
LU.AccessTy = getAccessType(LU.UserInst);
}
if (UI->isUseOfPostIncrementedValue())
LU.PostIncLoop = L;
const SCEV *S = IU.getCanonicalExpr(*UI);
// Equality (== and !=) ICmps are special. We can rewrite (i == N) as
// (N - i == 0), and this allows (N - i) to be the expression that we
// work with rather than just N or i, so we can consider the register
// requirements for both N and i at the same time. Limiting this code
// to equality icmps is not a problem because all interesting loops
// use equality icmps, thanks to IndVarSimplify.
if (ICmpInst *CI = dyn_cast<ICmpInst>(LU.UserInst))
if (CI->isEquality()) {
// Swap the operands if needed to put the OperandValToReplace on
// the left, for consistency.
Value *NV = CI->getOperand(1);
if (NV == LU.OperandValToReplace) {
CI->setOperand(1, CI->getOperand(0));
CI->setOperand(0, NV);
}
// x == y --> x - y == 0
const SCEV *N = SE.getSCEV(NV);
if (N->isLoopInvariant(L)) {
LU.Kind = LSRUse::ICmpZero;
S = SE.getMinusSCEV(N, S);
}
// -1 and the negations of all interesting strides (except the
// negation of -1) are now also interesting.
for (size_t i = 0, e = Factors.size(); i != e; ++i)
if (Factors[i] != -1)
Factors.insert(-(uint64_t)Factors[i]);
Factors.insert(-1);
}
// Ok, now enumerate all the different formulae we can find to compute
// the value for this expression.
LU.InsertInitialFormula(S, L, SE, DT);
CountRegisters(LU.Formulae.back(), Uses.size() - 1);
}
}
// If all uses use the same type, don't bother looking for truncation-based
// reuse.
if (Types.size() == 1)
Types.clear();
// Now use the reuse data to generate a bunch of interesting ways
// to formulate the values needed for the uses.
GenerateAllReuseFormulae();
// If there are any uses of registers that we're tracking that have escaped
// IVUsers' attention, add trivial uses for them, so that the register
// voting process takes the into consideration.
GenerateLoopInvariantRegisterUses();
// Sort the formulae. TODO: This is redundantly sorted below.
for (SmallVectorImpl<LSRUse>::iterator I = Uses.begin(), E = Uses.end();
I != E; ++I) {
LSRUse &LU = *I;
std::stable_sort(LU.Formulae.begin(), LU.Formulae.end(),
ComplexitySorter());
}
// Ok, we've now collected all the uses and noted their register uses. The
// next step is to start looking at register reuse possibilities.
DEBUG(print(dbgs()); dbgs() << '\n');
// Start by assuming we'll assign each use its own register. This is
// sometimes called "full" strength reduction, or "superhero" mode.
// Sometimes this is the best solution, but if there are opportunities for
// reuse we may find a better solution.
Score CurScore;
CurScore.RateInitial(Uses, L, SE);
// Create a sorted list of registers with those with the most uses appearing
// earlier in the list. We'll visit them first, as they're the most likely
// to represent profitable reuse opportunities.
SmallVector<RegCount, 8> RegOrder;
for (SmallVectorImpl<const SCEV *>::const_iterator I =
RegSequence.begin(), E = RegSequence.end(); I != E; ++I)
RegOrder.push_back(RegCount(*I, RegUses.find(*I)->second));
std::stable_sort(RegOrder.begin(), RegOrder.end());
// Visit each register. Determine which ones represent profitable reuse
// opportunities and remember them.
// TODO: Extract this code into a function.
for (SmallVectorImpl<RegCount>::const_iterator I = RegOrder.begin(),
E = RegOrder.end(); I != E; ++I) {
const SCEV *Reg = I->Reg;
const SmallBitVector &Bits = I->Sort.Bits;
// Registers with only one use don't represent reuse opportunities, so
// when we get there, we're done.
if (Bits.count() <= 1) break;
DEBUG(dbgs() << "Reg " << *Reg << ": ";
I->Sort.print(dbgs());
dbgs() << '\n');
// Determine the total number of registers will be needed if we make use
// of the reuse opportunity represented by the current register.
Score NewScore;
NewScore.Rate(Reg, Bits, Uses, L, SE);
// Now decide whether this register's reuse opportunity is an overall win.
// Currently the decision is heavily skewed for register pressure.
if (!(NewScore < CurScore)) {
continue;
}
// Ok, use this opportunity.
DEBUG(dbgs() << "This candidate has been accepted.\n");
CurScore = NewScore;
// Now that we've selected a new reuse opportunity, delete formulae that
// do not participate in that opportunity.
for (int j = Bits.find_first(); j != -1; j = Bits.find_next(j)) {
LSRUse &LU = Uses[j];
for (unsigned k = 0, h = LU.Formulae.size(); k != h; ++k) {
Formula &F = LU.Formulae[k];
if (!F.referencesReg(Reg)) {
std::swap(LU.Formulae[k], LU.Formulae.back());
LU.Formulae.pop_back();
--k; --h;
}
}
// Also re-sort the list to put the formulae with the fewest registers
// at the front.
// TODO: Do this earlier, we don't need it each time.
std::stable_sort(LU.Formulae.begin(), LU.Formulae.end(),
ComplexitySorter());
}
}
// Ok, we've now made all our decisions. The first formula for each use
// will be used.
DEBUG(dbgs() << "Concluding, we need "; CurScore.print(dbgs());
dbgs() << ".\n";
debug_winner(Uses));
// Free memory no longer needed.
RegOrder.clear();
Factors.clear();
Types.clear();
RegUses.clear();
RegSequence.clear();
// Keep track of instructions we may have made dead, so that
// we can remove them after we are done working.
SmallVector<WeakVH, 16> DeadInsts;
SCEVExpander Rewriter(SE);
Rewriter.disableCanonicalMode();
Rewriter.setIVIncInsertPos(L, IVIncInsertPos);
// Expand the new value definitions and update the users.
for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
E = Uses.end(); I != E; ++I) {
// Formulae should be legal.
DEBUG(for (SmallVectorImpl<Formula>::const_iterator J = I->Formulae.begin(),
JE = I->Formulae.end(); J != JE; ++J)
assert(isLegalUse(J->AM, I->Kind, I->AccessTy, TLI) &&
"Illegal formula generated!"));
// Expand the new code and update the user.
I->Rewrite(L, Rewriter, DeadInsts, SE, DT, P);
Changed = true;
}
// Clean up after ourselves. This must be done before deleting any
// instructions.
Rewriter.clear();
Changed |= DeleteTriviallyDeadInstructions(DeadInsts);
}
void LSRInstance::print(raw_ostream &OS) const {
OS << "LSR has identified the following interesting factors and types: ";
bool First = true;
for (SmallSetVector<int64_t, 4>::const_iterator
I = Factors.begin(), E = Factors.end(); I != E; ++I) {
if (!First) OS << ", ";
First = false;
OS << '*' << *I;
}
for (SmallSetVector<const Type *, 4>::const_iterator
I = Types.begin(), E = Types.end(); I != E; ++I) {
if (!First) OS << ", ";
First = false;
OS << '(' << **I << ')';
}
OS << '\n';
OS << "LSR is examining the following uses, and candidate formulae:\n";
for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
E = Uses.end(); I != E; ++I) {
const LSRUse &LU = *I;
dbgs() << " ";
LU.print(OS);
OS << '\n';
for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(),
JE = LU.Formulae.end(); J != JE; ++J) {
OS << " ";
J->print(OS);
OS << "\n";
}
}
}
void LSRInstance::dump() const {
print(errs()); errs() << '\n';
}
namespace {
class LoopStrengthReduce : public LoopPass {
/// TLI - Keep a pointer of a TargetLowering to consult for determining
/// transformation profitability.
const TargetLowering *const TLI;
public:
static char ID; // Pass ID, replacement for typeid
explicit LoopStrengthReduce(const TargetLowering *tli = NULL);
private:
bool runOnLoop(Loop *L, LPPassManager &LPM);
void getAnalysisUsage(AnalysisUsage &AU) const;
};
}
char LoopStrengthReduce::ID = 0;
static RegisterPass<LoopStrengthReduce>
X("loop-reduce", "Loop Strength Reduction");
Pass *llvm::createLoopStrengthReducePass(const TargetLowering *TLI) {
return new LoopStrengthReduce(TLI);
}
LoopStrengthReduce::LoopStrengthReduce(const TargetLowering *tli)
: LoopPass(&ID), TLI(tli) {}
void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
// We split critical edges, so we change the CFG. However, we do update
// many analyses if they are around.
AU.addPreservedID(LoopSimplifyID);
AU.addPreserved<LoopInfo>();
AU.addPreserved("domfrontier");
AU.addRequiredID(LoopSimplifyID);
AU.addRequired<DominatorTree>();
AU.addPreserved<DominatorTree>();
AU.addRequired<ScalarEvolution>();
AU.addPreserved<ScalarEvolution>();
AU.addRequired<IVUsers>();
AU.addPreserved<IVUsers>();
}
bool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
bool Changed = false;
// Run the main LSR transformation.
Changed |= LSRInstance(TLI, L, this).getChanged();
// At this point, it is worth checking to see if any recurrence PHIs are also
// dead, so that we can remove them as well.
Changed |= DeleteDeadPHIs(L->getHeader());
return Changed;
}