blob: 7686e65efed92fbbf3f428a432bb75f2e198a9bc [file] [log] [blame]
Sanjoy Das083f3892016-05-18 22:55:34 +00001//===- GuardWidening.cpp - ---- Guard widening ----------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the guard widening pass. The semantics of the
11// @llvm.experimental.guard intrinsic lets LLVM transform it so that it fails
12// more often that it did before the transform. This optimization is called
13// "widening" and can be used hoist and common runtime checks in situations like
14// these:
15//
16// %cmp0 = 7 u< Length
17// call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ]
18// call @unknown_side_effects()
19// %cmp1 = 9 u< Length
20// call @llvm.experimental.guard(i1 %cmp1) [ "deopt"(...) ]
21// ...
22//
23// =>
24//
25// %cmp0 = 9 u< Length
26// call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ]
27// call @unknown_side_effects()
28// ...
29//
30// If %cmp0 is false, @llvm.experimental.guard will "deoptimize" back to a
31// generic implementation of the same function, which will have the correct
32// semantics from that point onward. It is always _legal_ to deoptimize (so
33// replacing %cmp0 with false is "correct"), though it may not always be
34// profitable to do so.
35//
36// NB! This pass is a work in progress. It hasn't been tuned to be "production
37// ready" yet. It is known to have quadriatic running time and will not scale
38// to large numbers of guards
39//
40//===----------------------------------------------------------------------===//
41
42#include "llvm/Transforms/Scalar/GuardWidening.h"
43#include "llvm/Pass.h"
44#include "llvm/ADT/DenseMap.h"
45#include "llvm/ADT/DepthFirstIterator.h"
46#include "llvm/Analysis/LoopInfo.h"
47#include "llvm/Analysis/PostDominators.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/IR/Dominators.h"
50#include "llvm/IR/IntrinsicInst.h"
51#include "llvm/IR/PatternMatch.h"
52#include "llvm/Support/Debug.h"
53#include "llvm/Transforms/Scalar.h"
54
55using namespace llvm;
56
57#define DEBUG_TYPE "guard-widening"
58
59namespace {
60
61class GuardWideningImpl {
62 DominatorTree &DT;
63 PostDominatorTree &PDT;
64 LoopInfo &LI;
65
66 /// The set of guards whose conditions have been widened into dominating
67 /// guards.
68 SmallVector<IntrinsicInst *, 16> EliminatedGuards;
69
70 /// The set of guards which have been widened to include conditions to other
71 /// guards.
72 DenseSet<IntrinsicInst *> WidenedGuards;
73
74 /// Try to eliminate guard \p Guard by widening it into an earlier dominating
75 /// guard. \p DFSI is the DFS iterator on the dominator tree that is
76 /// currently visiting the block containing \p Guard, and \p GuardsPerBlock
77 /// maps BasicBlocks to the set of guards seen in that block.
78 bool eliminateGuardViaWidening(
79 IntrinsicInst *Guard, const df_iterator<DomTreeNode *> &DFSI,
80 const DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> &
81 GuardsPerBlock);
82
83 /// Used to keep track of which widening potential is more effective.
84 enum WideningScore {
85 /// Don't widen.
86 WS_IllegalOrNegative,
87
88 /// Widening is performance neutral as far as the cycles spent in check
89 /// conditions goes (but can still help, e.g., code layout, having less
90 /// deopt state).
91 WS_Neutral,
92
93 /// Widening is profitable.
94 WS_Positive,
95
96 /// Widening is very profitable. Not significantly different from \c
97 /// WS_Positive, except by the order.
98 WS_VeryPositive
99 };
100
101 static StringRef scoreTypeToString(WideningScore WS);
102
103 /// Compute the score for widening the condition in \p DominatedGuard
104 /// (contained in \p DominatedGuardLoop) into \p DominatingGuard (contained in
105 /// \p DominatingGuardLoop).
106 WideningScore computeWideningScore(IntrinsicInst *DominatedGuard,
107 Loop *DominatedGuardLoop,
108 IntrinsicInst *DominatingGuard,
109 Loop *DominatingGuardLoop);
110
111 /// Helper to check if \p V can be hoisted to \p InsertPos.
112 bool isAvailableAt(Value *V, Instruction *InsertPos) {
113 SmallPtrSet<Instruction *, 8> Visited;
114 return isAvailableAt(V, InsertPos, Visited);
115 }
116
117 bool isAvailableAt(Value *V, Instruction *InsertPos,
118 SmallPtrSetImpl<Instruction *> &Visited);
119
120 /// Helper to hoist \p V to \p InsertPos. Guaranteed to succeed if \c
121 /// isAvailableAt returned true.
122 void makeAvailableAt(Value *V, Instruction *InsertPos);
123
124 /// Common helper used by \c widenGuard and \c isWideningCondProfitable. Try
125 /// to generate an expression computing the logical AND of \p Cond0 and \p
126 /// Cond1. Return true if the expression computing the AND is only as
127 /// expensive as computing one of the two. If \p InsertPt is true then
128 /// actually generate the resulting expression, make it available at \p
129 /// InsertPt and return it in \p Result (else no change to the IR is made).
130 bool widenCondCommon(Value *Cond0, Value *Cond1, Instruction *InsertPt,
131 Value *&Result);
132
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000133 /// Represents a range check of the form \c Base + \c Offset u< \c Length,
134 /// with the constraint that \c Length is not negative. \c CheckInst is the
135 /// pre-existing instruction in the IR that computes the result of this range
136 /// check.
Sanjoy Dasbe991532016-05-24 20:54:45 +0000137 class RangeCheck {
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000138 Value *Base;
139 ConstantInt *Offset;
140 Value *Length;
141 ICmpInst *CheckInst;
142
Sanjoy Dasbe991532016-05-24 20:54:45 +0000143 public:
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000144 explicit RangeCheck(Value *Base, ConstantInt *Offset, Value *Length,
145 ICmpInst *CheckInst)
146 : Base(Base), Offset(Offset), Length(Length), CheckInst(CheckInst) {}
147
Sanjoy Dasbe991532016-05-24 20:54:45 +0000148 void setBase(Value *NewBase) { Base = NewBase; }
149 void setOffset(ConstantInt *NewOffset) { Offset = NewOffset; }
150
151 Value *getBase() const { return Base; }
152 ConstantInt *getOffset() const { return Offset; }
153 const APInt &getOffsetValue() const { return getOffset()->getValue(); }
154 Value *getLength() const { return Length; };
155 ICmpInst *getCheckInst() const { return CheckInst; }
156
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000157 void print(raw_ostream &OS, bool PrintTypes = false) {
158 OS << "Base: ";
159 Base->printAsOperand(OS, PrintTypes);
160 OS << " Offset: ";
161 Offset->printAsOperand(OS, PrintTypes);
162 OS << " Length: ";
163 Length->printAsOperand(OS, PrintTypes);
164 }
165
166 LLVM_DUMP_METHOD void dump() {
167 print(dbgs());
168 dbgs() << "\n";
169 }
170 };
171
172 /// Parse \p CheckCond into a conjunction (logical-and) of range checks; and
173 /// append them to \p Checks. Returns true on success, may clobber \c Checks
174 /// on failure.
175 bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks) {
176 SmallPtrSet<Value *, 8> Visited;
177 return parseRangeChecks(CheckCond, Checks, Visited);
178 }
179
180 bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks,
181 SmallPtrSetImpl<Value *> &Visited);
182
183 /// Combine the checks in \p Checks into a smaller set of checks and append
184 /// them into \p CombinedChecks. Return true on success (i.e. all of checks
185 /// in \p Checks were combined into \p CombinedChecks). Clobbers \p Checks
186 /// and \p CombinedChecks on success and on failure.
187 bool combineRangeChecks(SmallVectorImpl<RangeCheck> &Checks,
188 SmallVectorImpl<RangeCheck> &CombinedChecks);
189
Sanjoy Das083f3892016-05-18 22:55:34 +0000190 /// Can we compute the logical AND of \p Cond0 and \p Cond1 for the price of
191 /// computing only one of the two expressions?
192 bool isWideningCondProfitable(Value *Cond0, Value *Cond1) {
193 Value *ResultUnused;
194 return widenCondCommon(Cond0, Cond1, /*InsertPt=*/nullptr, ResultUnused);
195 }
196
197 /// Widen \p ToWiden to fail if \p NewCondition is false (in addition to
198 /// whatever it is already checking).
199 void widenGuard(IntrinsicInst *ToWiden, Value *NewCondition) {
200 Value *Result;
201 widenCondCommon(ToWiden->getArgOperand(0), NewCondition, ToWiden, Result);
202 ToWiden->setArgOperand(0, Result);
203 }
204
205public:
206 explicit GuardWideningImpl(DominatorTree &DT, PostDominatorTree &PDT,
207 LoopInfo &LI)
208 : DT(DT), PDT(PDT), LI(LI) {}
209
210 /// The entry point for this pass.
211 bool run();
212};
213
214struct GuardWideningLegacyPass : public FunctionPass {
215 static char ID;
216 GuardWideningPass Impl;
217
218 GuardWideningLegacyPass() : FunctionPass(ID) {
219 initializeGuardWideningLegacyPassPass(*PassRegistry::getPassRegistry());
220 }
221
222 bool runOnFunction(Function &F) override {
223 if (skipFunction(F))
224 return false;
225 return GuardWideningImpl(
226 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
227 getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree(),
228 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()).run();
229 }
230
231 void getAnalysisUsage(AnalysisUsage &AU) const override {
232 AU.setPreservesCFG();
233 AU.addRequired<DominatorTreeWrapperPass>();
234 AU.addRequired<PostDominatorTreeWrapperPass>();
235 AU.addRequired<LoopInfoWrapperPass>();
236 }
237};
238
239}
240
241bool GuardWideningImpl::run() {
242 using namespace llvm::PatternMatch;
243
244 DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> GuardsInBlock;
245 bool Changed = false;
246
247 for (auto DFI = df_begin(DT.getRootNode()), DFE = df_end(DT.getRootNode());
248 DFI != DFE; ++DFI) {
249 auto *BB = (*DFI)->getBlock();
250 auto &CurrentList = GuardsInBlock[BB];
251
252 for (auto &I : *BB)
253 if (match(&I, m_Intrinsic<Intrinsic::experimental_guard>()))
254 CurrentList.push_back(cast<IntrinsicInst>(&I));
255
256 for (auto *II : CurrentList)
257 Changed |= eliminateGuardViaWidening(II, DFI, GuardsInBlock);
258 }
259
260 for (auto *II : EliminatedGuards)
261 if (!WidenedGuards.count(II))
262 II->eraseFromParent();
263
264 return Changed;
265}
266
267bool GuardWideningImpl::eliminateGuardViaWidening(
268 IntrinsicInst *GuardInst, const df_iterator<DomTreeNode *> &DFSI,
269 const DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> &
270 GuardsInBlock) {
271 IntrinsicInst *BestSoFar = nullptr;
272 auto BestScoreSoFar = WS_IllegalOrNegative;
273 auto *GuardInstLoop = LI.getLoopFor(GuardInst->getParent());
274
275 // In the set of dominating guards, find the one we can merge GuardInst with
276 // for the most profit.
277 for (unsigned i = 0, e = DFSI.getPathLength(); i != e; ++i) {
278 auto *CurBB = DFSI.getPath(i)->getBlock();
279 auto *CurLoop = LI.getLoopFor(CurBB);
280 assert(GuardsInBlock.count(CurBB) && "Must have been populated by now!");
281 const auto &GuardsInCurBB = GuardsInBlock.find(CurBB)->second;
282
283 auto I = GuardsInCurBB.begin();
284 auto E = GuardsInCurBB.end();
285
286#ifndef NDEBUG
287 {
288 unsigned Index = 0;
289 for (auto &I : *CurBB) {
290 if (Index == GuardsInCurBB.size())
291 break;
292 if (GuardsInCurBB[Index] == &I)
293 Index++;
294 }
295 assert(Index == GuardsInCurBB.size() &&
296 "Guards expected to be in order!");
297 }
298#endif
299
300 assert((i == (e - 1)) == (GuardInst->getParent() == CurBB) && "Bad DFS?");
301
302 if (i == (e - 1)) {
303 // Corner case: make sure we're only looking at guards strictly dominating
304 // GuardInst when visiting GuardInst->getParent().
305 auto NewEnd = std::find(I, E, GuardInst);
306 assert(NewEnd != E && "GuardInst not in its own block?");
307 E = NewEnd;
308 }
309
310 for (auto *Candidate : make_range(I, E)) {
311 auto Score =
312 computeWideningScore(GuardInst, GuardInstLoop, Candidate, CurLoop);
313 DEBUG(dbgs() << "Score between " << *GuardInst->getArgOperand(0)
314 << " and " << *Candidate->getArgOperand(0) << " is "
315 << scoreTypeToString(Score) << "\n");
316 if (Score > BestScoreSoFar) {
317 BestScoreSoFar = Score;
318 BestSoFar = Candidate;
319 }
320 }
321 }
322
323 if (BestScoreSoFar == WS_IllegalOrNegative) {
324 DEBUG(dbgs() << "Did not eliminate guard " << *GuardInst << "\n");
325 return false;
326 }
327
328 assert(BestSoFar != GuardInst && "Should have never visited same guard!");
329 assert(DT.dominates(BestSoFar, GuardInst) && "Should be!");
330
331 DEBUG(dbgs() << "Widening " << *GuardInst << " into " << *BestSoFar
332 << " with score " << scoreTypeToString(BestScoreSoFar) << "\n");
333 widenGuard(BestSoFar, GuardInst->getArgOperand(0));
334 GuardInst->setArgOperand(0, ConstantInt::getTrue(GuardInst->getContext()));
335 EliminatedGuards.push_back(GuardInst);
336 WidenedGuards.insert(BestSoFar);
337 return true;
338}
339
340GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
341 IntrinsicInst *DominatedGuard, Loop *DominatedGuardLoop,
342 IntrinsicInst *DominatingGuard, Loop *DominatingGuardLoop) {
343 bool HoistingOutOfLoop = false;
344
345 if (DominatingGuardLoop != DominatedGuardLoop) {
346 if (DominatingGuardLoop &&
347 !DominatingGuardLoop->contains(DominatedGuardLoop))
348 return WS_IllegalOrNegative;
349
350 HoistingOutOfLoop = true;
351 }
352
353 if (!isAvailableAt(DominatedGuard->getArgOperand(0), DominatingGuard))
354 return WS_IllegalOrNegative;
355
356 bool HoistingOutOfIf =
357 !PDT.dominates(DominatedGuard->getParent(), DominatingGuard->getParent());
358
359 if (isWideningCondProfitable(DominatedGuard->getArgOperand(0),
360 DominatingGuard->getArgOperand(0)))
361 return HoistingOutOfLoop ? WS_VeryPositive : WS_Positive;
362
363 if (HoistingOutOfLoop)
364 return WS_Positive;
365
366 return HoistingOutOfIf ? WS_IllegalOrNegative : WS_Neutral;
367}
368
369bool GuardWideningImpl::isAvailableAt(Value *V, Instruction *Loc,
370 SmallPtrSetImpl<Instruction *> &Visited) {
371 auto *Inst = dyn_cast<Instruction>(V);
372 if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst))
373 return true;
374
375 if (!isSafeToSpeculativelyExecute(Inst, Loc, &DT) ||
376 Inst->mayReadFromMemory())
377 return false;
378
379 Visited.insert(Inst);
380
381 // We only want to go _up_ the dominance chain when recursing.
382 assert(!isa<PHINode>(Loc) &&
383 "PHIs should return false for isSafeToSpeculativelyExecute");
384 assert(DT.isReachableFromEntry(Inst->getParent()) &&
385 "We did a DFS from the block entry!");
386 return all_of(Inst->operands(),
387 [&](Value *Op) { return isAvailableAt(Op, Loc, Visited); });
388}
389
390void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) {
391 auto *Inst = dyn_cast<Instruction>(V);
392 if (!Inst || DT.dominates(Inst, Loc))
393 return;
394
395 assert(isSafeToSpeculativelyExecute(Inst, Loc, &DT) &&
396 !Inst->mayReadFromMemory() && "Should've checked with isAvailableAt!");
397
398 for (Value *Op : Inst->operands())
399 makeAvailableAt(Op, Loc);
400
401 Inst->moveBefore(Loc);
402}
403
404bool GuardWideningImpl::widenCondCommon(Value *Cond0, Value *Cond1,
405 Instruction *InsertPt, Value *&Result) {
406 using namespace llvm::PatternMatch;
407
408 {
409 // L >u C0 && L >u C1 -> L >u max(C0, C1)
410 ConstantInt *RHS0, *RHS1;
411 Value *LHS;
412 ICmpInst::Predicate Pred0, Pred1;
413 if (match(Cond0, m_ICmp(Pred0, m_Value(LHS), m_ConstantInt(RHS0))) &&
414 match(Cond1, m_ICmp(Pred1, m_Specific(LHS), m_ConstantInt(RHS1)))) {
415
Sanjoy Dasb784ed32016-05-19 03:53:17 +0000416 ConstantRange CR0 =
417 ConstantRange::makeExactICmpRegion(Pred0, RHS0->getValue());
418 ConstantRange CR1 =
419 ConstantRange::makeExactICmpRegion(Pred1, RHS1->getValue());
Sanjoy Das083f3892016-05-18 22:55:34 +0000420
Sanjoy Dasb784ed32016-05-19 03:53:17 +0000421 // SubsetIntersect is a subset of the actual mathematical intersection of
Sanjay Patelf8ee0e02016-06-19 17:20:27 +0000422 // CR0 and CR1, while SupersetIntersect is a superset of the actual
Sanjoy Dasb784ed32016-05-19 03:53:17 +0000423 // mathematical intersection. If these two ConstantRanges are equal, then
424 // we know we were able to represent the actual mathematical intersection
425 // of CR0 and CR1, and can use the same to generate an icmp instruction.
426 //
427 // Given what we're doing here and the semantics of guards, it would
428 // actually be correct to just use SubsetIntersect, but that may be too
429 // aggressive in cases we care about.
430 auto SubsetIntersect = CR0.inverse().unionWith(CR1.inverse()).inverse();
431 auto SupersetIntersect = CR0.intersectWith(CR1);
432
433 APInt NewRHSAP;
434 CmpInst::Predicate Pred;
435 if (SubsetIntersect == SupersetIntersect &&
436 SubsetIntersect.getEquivalentICmp(Pred, NewRHSAP)) {
Sanjoy Das083f3892016-05-18 22:55:34 +0000437 if (InsertPt) {
Sanjoy Dasb784ed32016-05-19 03:53:17 +0000438 ConstantInt *NewRHS = ConstantInt::get(Cond0->getContext(), NewRHSAP);
439 Result = new ICmpInst(InsertPt, Pred, LHS, NewRHS, "wide.chk");
Sanjoy Das083f3892016-05-18 22:55:34 +0000440 }
Sanjoy Das083f3892016-05-18 22:55:34 +0000441 return true;
442 }
443 }
444 }
445
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000446 {
447 SmallVector<GuardWideningImpl::RangeCheck, 4> Checks, CombinedChecks;
448 if (parseRangeChecks(Cond0, Checks) && parseRangeChecks(Cond1, Checks) &&
449 combineRangeChecks(Checks, CombinedChecks)) {
450 if (InsertPt) {
451 Result = nullptr;
452 for (auto &RC : CombinedChecks) {
Sanjoy Dasbe991532016-05-24 20:54:45 +0000453 makeAvailableAt(RC.getCheckInst(), InsertPt);
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000454 if (Result)
Sanjoy Dasbe991532016-05-24 20:54:45 +0000455 Result = BinaryOperator::CreateAnd(RC.getCheckInst(), Result, "",
456 InsertPt);
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000457 else
Sanjoy Dasbe991532016-05-24 20:54:45 +0000458 Result = RC.getCheckInst();
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000459 }
460
461 Result->setName("wide.chk");
462 }
463 return true;
464 }
465 }
466
Sanjoy Das083f3892016-05-18 22:55:34 +0000467 // Base case -- just logical-and the two conditions together.
468
469 if (InsertPt) {
470 makeAvailableAt(Cond0, InsertPt);
471 makeAvailableAt(Cond1, InsertPt);
472
473 Result = BinaryOperator::CreateAnd(Cond0, Cond1, "wide.chk", InsertPt);
474 }
475
476 // We were not able to compute Cond0 AND Cond1 for the price of one.
477 return false;
478}
479
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000480bool GuardWideningImpl::parseRangeChecks(
481 Value *CheckCond, SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
482 SmallPtrSetImpl<Value *> &Visited) {
483 if (!Visited.insert(CheckCond).second)
484 return true;
485
486 using namespace llvm::PatternMatch;
487
488 {
489 Value *AndLHS, *AndRHS;
490 if (match(CheckCond, m_And(m_Value(AndLHS), m_Value(AndRHS))))
491 return parseRangeChecks(AndLHS, Checks) &&
492 parseRangeChecks(AndRHS, Checks);
493 }
494
495 auto *IC = dyn_cast<ICmpInst>(CheckCond);
496 if (!IC || !IC->getOperand(0)->getType()->isIntegerTy() ||
497 (IC->getPredicate() != ICmpInst::ICMP_ULT &&
498 IC->getPredicate() != ICmpInst::ICMP_UGT))
499 return false;
500
501 Value *CmpLHS = IC->getOperand(0), *CmpRHS = IC->getOperand(1);
502 if (IC->getPredicate() == ICmpInst::ICMP_UGT)
503 std::swap(CmpLHS, CmpRHS);
504
505 auto &DL = IC->getModule()->getDataLayout();
506
Sanjoy Dasbe991532016-05-24 20:54:45 +0000507 GuardWideningImpl::RangeCheck Check(
508 CmpLHS, cast<ConstantInt>(ConstantInt::getNullValue(CmpRHS->getType())),
509 CmpRHS, IC);
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000510
Sanjoy Dasbe991532016-05-24 20:54:45 +0000511 if (!isKnownNonNegative(Check.getLength(), DL))
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000512 return false;
513
514 // What we have in \c Check now is a correct interpretation of \p CheckCond.
515 // Try to see if we can move some constant offsets into the \c Offset field.
516
517 bool Changed;
Sanjoy Dasbe991532016-05-24 20:54:45 +0000518 auto &Ctx = CheckCond->getContext();
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000519
520 do {
521 Value *OpLHS;
522 ConstantInt *OpRHS;
523 Changed = false;
524
525#ifndef NDEBUG
Sanjoy Dasbe991532016-05-24 20:54:45 +0000526 auto *BaseInst = dyn_cast<Instruction>(Check.getBase());
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000527 assert((!BaseInst || DT.isReachableFromEntry(BaseInst->getParent())) &&
528 "Unreachable instruction?");
529#endif
530
Sanjoy Dasbe991532016-05-24 20:54:45 +0000531 if (match(Check.getBase(), m_Add(m_Value(OpLHS), m_ConstantInt(OpRHS)))) {
532 Check.setBase(OpLHS);
533 APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue();
534 Check.setOffset(ConstantInt::get(Ctx, NewOffset));
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000535 Changed = true;
Sanjoy Dasbe991532016-05-24 20:54:45 +0000536 } else if (match(Check.getBase(),
537 m_Or(m_Value(OpLHS), m_ConstantInt(OpRHS)))) {
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000538 unsigned BitWidth = OpLHS->getType()->getScalarSizeInBits();
539 APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
540 computeKnownBits(OpLHS, KnownZero, KnownOne, DL);
541 if ((OpRHS->getValue() & KnownZero) == OpRHS->getValue()) {
Sanjoy Dasbe991532016-05-24 20:54:45 +0000542 Check.setBase(OpLHS);
543 APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue();
544 Check.setOffset(ConstantInt::get(Ctx, NewOffset));
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000545 Changed = true;
546 }
547 }
548 } while (Changed);
549
550 Checks.push_back(Check);
551 return true;
552}
553
554bool GuardWideningImpl::combineRangeChecks(
555 SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
556 SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) {
557 unsigned OldCount = Checks.size();
558 while (!Checks.empty()) {
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000559 // Pick all of the range checks with a specific base and length, and try to
560 // merge them.
Sanjoy Dasbe991532016-05-24 20:54:45 +0000561 Value *CurrentBase = Checks.front().getBase();
562 Value *CurrentLength = Checks.front().getLength();
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000563
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000564 SmallVector<GuardWideningImpl::RangeCheck, 3> CurrentChecks;
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000565
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000566 auto IsCurrentCheck = [&](GuardWideningImpl::RangeCheck &RC) {
Sanjoy Dasbe991532016-05-24 20:54:45 +0000567 return RC.getBase() == CurrentBase && RC.getLength() == CurrentLength;
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000568 };
569
570 std::copy_if(Checks.begin(), Checks.end(),
571 std::back_inserter(CurrentChecks), IsCurrentCheck);
572 Checks.erase(remove_if(Checks, IsCurrentCheck), Checks.end());
573
574 assert(CurrentChecks.size() != 0 && "We know we have at least one!");
575
576 if (CurrentChecks.size() < 3) {
577 RangeChecksOut.insert(RangeChecksOut.end(), CurrentChecks.begin(),
578 CurrentChecks.end());
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000579 continue;
580 }
581
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000582 // CurrentChecks.size() will typically be 3 here, but so far there has been
583 // no need to hard-code that fact.
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000584
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000585 std::sort(CurrentChecks.begin(), CurrentChecks.end(),
Sanjoy Das23519752016-05-19 23:15:59 +0000586 [&](const GuardWideningImpl::RangeCheck &LHS,
587 const GuardWideningImpl::RangeCheck &RHS) {
Sanjoy Dasbe991532016-05-24 20:54:45 +0000588 return LHS.getOffsetValue().slt(RHS.getOffsetValue());
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000589 });
590
591 // Note: std::sort should not invalidate the ChecksStart iterator.
592
Sanjoy Dasbe991532016-05-24 20:54:45 +0000593 ConstantInt *MinOffset = CurrentChecks.front().getOffset(),
594 *MaxOffset = CurrentChecks.back().getOffset();
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000595
596 unsigned BitWidth = MaxOffset->getValue().getBitWidth();
597 if ((MaxOffset->getValue() - MinOffset->getValue())
598 .ugt(APInt::getSignedMinValue(BitWidth)))
599 return false;
600
601 APInt MaxDiff = MaxOffset->getValue() - MinOffset->getValue();
Benjamin Kramer46e38f32016-06-08 10:01:20 +0000602 const APInt &HighOffset = MaxOffset->getValue();
Sanjoy Das23519752016-05-19 23:15:59 +0000603 auto OffsetOK = [&](const GuardWideningImpl::RangeCheck &RC) {
Sanjoy Dasbe991532016-05-24 20:54:45 +0000604 return (HighOffset - RC.getOffsetValue()).ult(MaxDiff);
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000605 };
606
607 if (MaxDiff.isMinValue() ||
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000608 !std::all_of(std::next(CurrentChecks.begin()), CurrentChecks.end(),
609 OffsetOK))
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000610 return false;
611
612 // We have a series of f+1 checks as:
613 //
614 // I+k_0 u< L ... Chk_0
615 // I_k_1 u< L ... Chk_1
616 // ...
617 // I_k_f u< L ... Chk_(f+1)
618 //
619 // with forall i in [0,f): k_f-k_i u< k_f-k_0 ... Precond_0
620 // k_f-k_0 u< INT_MIN+k_f ... Precond_1
621 // k_f != k_0 ... Precond_2
622 //
623 // Claim:
624 // Chk_0 AND Chk_(f+1) implies all the other checks
625 //
626 // Informal proof sketch:
627 //
628 // We will show that the integer range [I+k_0,I+k_f] does not unsigned-wrap
629 // (i.e. going from I+k_0 to I+k_f does not cross the -1,0 boundary) and
630 // thus I+k_f is the greatest unsigned value in that range.
631 //
632 // This combined with Ckh_(f+1) shows that everything in that range is u< L.
633 // Via Precond_0 we know that all of the indices in Chk_0 through Chk_(f+1)
634 // lie in [I+k_0,I+k_f], this proving our claim.
635 //
636 // To see that [I+k_0,I+k_f] is not a wrapping range, note that there are
637 // two possibilities: I+k_0 u< I+k_f or I+k_0 >u I+k_f (they can't be equal
638 // since k_0 != k_f). In the former case, [I+k_0,I+k_f] is not a wrapping
639 // range by definition, and the latter case is impossible:
640 //
641 // 0-----I+k_f---I+k_0----L---INT_MAX,INT_MIN------------------(-1)
642 // xxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
643 //
644 // For Chk_0 to succeed, we'd have to have k_f-k_0 (the range highlighted
645 // with 'x' above) to be at least >u INT_MIN.
646
Sanjoy Dasbe6c7a12016-05-21 02:24:44 +0000647 RangeChecksOut.emplace_back(CurrentChecks.front());
648 RangeChecksOut.emplace_back(CurrentChecks.back());
Sanjoy Dasf5f03312016-05-19 22:55:46 +0000649 }
650
651 assert(RangeChecksOut.size() <= OldCount && "We pessimized!");
652 return RangeChecksOut.size() != OldCount;
653}
654
Sanjoy Das083f3892016-05-18 22:55:34 +0000655PreservedAnalyses GuardWideningPass::run(Function &F,
656 AnalysisManager<Function> &AM) {
657 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
658 auto &LI = AM.getResult<LoopAnalysis>(F);
659 auto &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
660 bool Changed = GuardWideningImpl(DT, PDT, LI).run();
661 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
662}
663
664StringRef GuardWideningImpl::scoreTypeToString(WideningScore WS) {
665 switch (WS) {
666 case WS_IllegalOrNegative:
667 return "IllegalOrNegative";
668 case WS_Neutral:
669 return "Neutral";
670 case WS_Positive:
671 return "Positive";
672 case WS_VeryPositive:
673 return "VeryPositive";
674 }
675
676 llvm_unreachable("Fully covered switch above!");
677}
678
679char GuardWideningLegacyPass::ID = 0;
680
681INITIALIZE_PASS_BEGIN(GuardWideningLegacyPass, "guard-widening", "Widen guards",
682 false, false)
683INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
684INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
685INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
686INITIALIZE_PASS_END(GuardWideningLegacyPass, "guard-widening", "Widen guards",
687 false, false)
688
689FunctionPass *llvm::createGuardWideningPass() {
690 return new GuardWideningLegacyPass();
691}