Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 1 | //===- GuardWidening.cpp - ---- Guard widening ----------------------------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file implements the guard widening pass. The semantics of the |
| 11 | // @llvm.experimental.guard intrinsic lets LLVM transform it so that it fails |
| 12 | // more often that it did before the transform. This optimization is called |
| 13 | // "widening" and can be used hoist and common runtime checks in situations like |
| 14 | // these: |
| 15 | // |
| 16 | // %cmp0 = 7 u< Length |
| 17 | // call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ] |
| 18 | // call @unknown_side_effects() |
| 19 | // %cmp1 = 9 u< Length |
| 20 | // call @llvm.experimental.guard(i1 %cmp1) [ "deopt"(...) ] |
| 21 | // ... |
| 22 | // |
| 23 | // => |
| 24 | // |
| 25 | // %cmp0 = 9 u< Length |
| 26 | // call @llvm.experimental.guard(i1 %cmp0) [ "deopt"(...) ] |
| 27 | // call @unknown_side_effects() |
| 28 | // ... |
| 29 | // |
| 30 | // If %cmp0 is false, @llvm.experimental.guard will "deoptimize" back to a |
| 31 | // generic implementation of the same function, which will have the correct |
| 32 | // semantics from that point onward. It is always _legal_ to deoptimize (so |
| 33 | // replacing %cmp0 with false is "correct"), though it may not always be |
| 34 | // profitable to do so. |
| 35 | // |
| 36 | // NB! This pass is a work in progress. It hasn't been tuned to be "production |
| 37 | // ready" yet. It is known to have quadriatic running time and will not scale |
| 38 | // to large numbers of guards |
| 39 | // |
| 40 | //===----------------------------------------------------------------------===// |
| 41 | |
| 42 | #include "llvm/Transforms/Scalar/GuardWidening.h" |
| 43 | #include "llvm/Pass.h" |
| 44 | #include "llvm/ADT/DenseMap.h" |
| 45 | #include "llvm/ADT/DepthFirstIterator.h" |
| 46 | #include "llvm/Analysis/LoopInfo.h" |
| 47 | #include "llvm/Analysis/PostDominators.h" |
| 48 | #include "llvm/Analysis/ValueTracking.h" |
| 49 | #include "llvm/IR/Dominators.h" |
| 50 | #include "llvm/IR/IntrinsicInst.h" |
| 51 | #include "llvm/IR/PatternMatch.h" |
| 52 | #include "llvm/Support/Debug.h" |
| 53 | #include "llvm/Transforms/Scalar.h" |
| 54 | |
| 55 | using namespace llvm; |
| 56 | |
| 57 | #define DEBUG_TYPE "guard-widening" |
| 58 | |
| 59 | namespace { |
| 60 | |
| 61 | class GuardWideningImpl { |
| 62 | DominatorTree &DT; |
| 63 | PostDominatorTree &PDT; |
| 64 | LoopInfo &LI; |
| 65 | |
| 66 | /// The set of guards whose conditions have been widened into dominating |
| 67 | /// guards. |
| 68 | SmallVector<IntrinsicInst *, 16> EliminatedGuards; |
| 69 | |
| 70 | /// The set of guards which have been widened to include conditions to other |
| 71 | /// guards. |
| 72 | DenseSet<IntrinsicInst *> WidenedGuards; |
| 73 | |
| 74 | /// Try to eliminate guard \p Guard by widening it into an earlier dominating |
| 75 | /// guard. \p DFSI is the DFS iterator on the dominator tree that is |
| 76 | /// currently visiting the block containing \p Guard, and \p GuardsPerBlock |
| 77 | /// maps BasicBlocks to the set of guards seen in that block. |
| 78 | bool eliminateGuardViaWidening( |
| 79 | IntrinsicInst *Guard, const df_iterator<DomTreeNode *> &DFSI, |
| 80 | const DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> & |
| 81 | GuardsPerBlock); |
| 82 | |
| 83 | /// Used to keep track of which widening potential is more effective. |
| 84 | enum WideningScore { |
| 85 | /// Don't widen. |
| 86 | WS_IllegalOrNegative, |
| 87 | |
| 88 | /// Widening is performance neutral as far as the cycles spent in check |
| 89 | /// conditions goes (but can still help, e.g., code layout, having less |
| 90 | /// deopt state). |
| 91 | WS_Neutral, |
| 92 | |
| 93 | /// Widening is profitable. |
| 94 | WS_Positive, |
| 95 | |
| 96 | /// Widening is very profitable. Not significantly different from \c |
| 97 | /// WS_Positive, except by the order. |
| 98 | WS_VeryPositive |
| 99 | }; |
| 100 | |
| 101 | static StringRef scoreTypeToString(WideningScore WS); |
| 102 | |
| 103 | /// Compute the score for widening the condition in \p DominatedGuard |
| 104 | /// (contained in \p DominatedGuardLoop) into \p DominatingGuard (contained in |
| 105 | /// \p DominatingGuardLoop). |
| 106 | WideningScore computeWideningScore(IntrinsicInst *DominatedGuard, |
| 107 | Loop *DominatedGuardLoop, |
| 108 | IntrinsicInst *DominatingGuard, |
| 109 | Loop *DominatingGuardLoop); |
| 110 | |
| 111 | /// Helper to check if \p V can be hoisted to \p InsertPos. |
| 112 | bool isAvailableAt(Value *V, Instruction *InsertPos) { |
| 113 | SmallPtrSet<Instruction *, 8> Visited; |
| 114 | return isAvailableAt(V, InsertPos, Visited); |
| 115 | } |
| 116 | |
| 117 | bool isAvailableAt(Value *V, Instruction *InsertPos, |
| 118 | SmallPtrSetImpl<Instruction *> &Visited); |
| 119 | |
| 120 | /// Helper to hoist \p V to \p InsertPos. Guaranteed to succeed if \c |
| 121 | /// isAvailableAt returned true. |
| 122 | void makeAvailableAt(Value *V, Instruction *InsertPos); |
| 123 | |
| 124 | /// Common helper used by \c widenGuard and \c isWideningCondProfitable. Try |
| 125 | /// to generate an expression computing the logical AND of \p Cond0 and \p |
| 126 | /// Cond1. Return true if the expression computing the AND is only as |
| 127 | /// expensive as computing one of the two. If \p InsertPt is true then |
| 128 | /// actually generate the resulting expression, make it available at \p |
| 129 | /// InsertPt and return it in \p Result (else no change to the IR is made). |
| 130 | bool widenCondCommon(Value *Cond0, Value *Cond1, Instruction *InsertPt, |
| 131 | Value *&Result); |
| 132 | |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 133 | /// Represents a range check of the form \c Base + \c Offset u< \c Length, |
| 134 | /// with the constraint that \c Length is not negative. \c CheckInst is the |
| 135 | /// pre-existing instruction in the IR that computes the result of this range |
| 136 | /// check. |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 137 | class RangeCheck { |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 138 | Value *Base; |
| 139 | ConstantInt *Offset; |
| 140 | Value *Length; |
| 141 | ICmpInst *CheckInst; |
| 142 | |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 143 | public: |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 144 | explicit RangeCheck(Value *Base, ConstantInt *Offset, Value *Length, |
| 145 | ICmpInst *CheckInst) |
| 146 | : Base(Base), Offset(Offset), Length(Length), CheckInst(CheckInst) {} |
| 147 | |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 148 | void setBase(Value *NewBase) { Base = NewBase; } |
| 149 | void setOffset(ConstantInt *NewOffset) { Offset = NewOffset; } |
| 150 | |
| 151 | Value *getBase() const { return Base; } |
| 152 | ConstantInt *getOffset() const { return Offset; } |
| 153 | const APInt &getOffsetValue() const { return getOffset()->getValue(); } |
| 154 | Value *getLength() const { return Length; }; |
| 155 | ICmpInst *getCheckInst() const { return CheckInst; } |
| 156 | |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 157 | void print(raw_ostream &OS, bool PrintTypes = false) { |
| 158 | OS << "Base: "; |
| 159 | Base->printAsOperand(OS, PrintTypes); |
| 160 | OS << " Offset: "; |
| 161 | Offset->printAsOperand(OS, PrintTypes); |
| 162 | OS << " Length: "; |
| 163 | Length->printAsOperand(OS, PrintTypes); |
| 164 | } |
| 165 | |
| 166 | LLVM_DUMP_METHOD void dump() { |
| 167 | print(dbgs()); |
| 168 | dbgs() << "\n"; |
| 169 | } |
| 170 | }; |
| 171 | |
| 172 | /// Parse \p CheckCond into a conjunction (logical-and) of range checks; and |
| 173 | /// append them to \p Checks. Returns true on success, may clobber \c Checks |
| 174 | /// on failure. |
| 175 | bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks) { |
| 176 | SmallPtrSet<Value *, 8> Visited; |
| 177 | return parseRangeChecks(CheckCond, Checks, Visited); |
| 178 | } |
| 179 | |
| 180 | bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks, |
| 181 | SmallPtrSetImpl<Value *> &Visited); |
| 182 | |
| 183 | /// Combine the checks in \p Checks into a smaller set of checks and append |
| 184 | /// them into \p CombinedChecks. Return true on success (i.e. all of checks |
| 185 | /// in \p Checks were combined into \p CombinedChecks). Clobbers \p Checks |
| 186 | /// and \p CombinedChecks on success and on failure. |
| 187 | bool combineRangeChecks(SmallVectorImpl<RangeCheck> &Checks, |
| 188 | SmallVectorImpl<RangeCheck> &CombinedChecks); |
| 189 | |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 190 | /// Can we compute the logical AND of \p Cond0 and \p Cond1 for the price of |
| 191 | /// computing only one of the two expressions? |
| 192 | bool isWideningCondProfitable(Value *Cond0, Value *Cond1) { |
| 193 | Value *ResultUnused; |
| 194 | return widenCondCommon(Cond0, Cond1, /*InsertPt=*/nullptr, ResultUnused); |
| 195 | } |
| 196 | |
| 197 | /// Widen \p ToWiden to fail if \p NewCondition is false (in addition to |
| 198 | /// whatever it is already checking). |
| 199 | void widenGuard(IntrinsicInst *ToWiden, Value *NewCondition) { |
| 200 | Value *Result; |
| 201 | widenCondCommon(ToWiden->getArgOperand(0), NewCondition, ToWiden, Result); |
| 202 | ToWiden->setArgOperand(0, Result); |
| 203 | } |
| 204 | |
| 205 | public: |
| 206 | explicit GuardWideningImpl(DominatorTree &DT, PostDominatorTree &PDT, |
| 207 | LoopInfo &LI) |
| 208 | : DT(DT), PDT(PDT), LI(LI) {} |
| 209 | |
| 210 | /// The entry point for this pass. |
| 211 | bool run(); |
| 212 | }; |
| 213 | |
| 214 | struct GuardWideningLegacyPass : public FunctionPass { |
| 215 | static char ID; |
| 216 | GuardWideningPass Impl; |
| 217 | |
| 218 | GuardWideningLegacyPass() : FunctionPass(ID) { |
| 219 | initializeGuardWideningLegacyPassPass(*PassRegistry::getPassRegistry()); |
| 220 | } |
| 221 | |
| 222 | bool runOnFunction(Function &F) override { |
| 223 | if (skipFunction(F)) |
| 224 | return false; |
| 225 | return GuardWideningImpl( |
| 226 | getAnalysis<DominatorTreeWrapperPass>().getDomTree(), |
| 227 | getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree(), |
| 228 | getAnalysis<LoopInfoWrapperPass>().getLoopInfo()).run(); |
| 229 | } |
| 230 | |
| 231 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 232 | AU.setPreservesCFG(); |
| 233 | AU.addRequired<DominatorTreeWrapperPass>(); |
| 234 | AU.addRequired<PostDominatorTreeWrapperPass>(); |
| 235 | AU.addRequired<LoopInfoWrapperPass>(); |
| 236 | } |
| 237 | }; |
| 238 | |
| 239 | } |
| 240 | |
| 241 | bool GuardWideningImpl::run() { |
| 242 | using namespace llvm::PatternMatch; |
| 243 | |
| 244 | DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> GuardsInBlock; |
| 245 | bool Changed = false; |
| 246 | |
| 247 | for (auto DFI = df_begin(DT.getRootNode()), DFE = df_end(DT.getRootNode()); |
| 248 | DFI != DFE; ++DFI) { |
| 249 | auto *BB = (*DFI)->getBlock(); |
| 250 | auto &CurrentList = GuardsInBlock[BB]; |
| 251 | |
| 252 | for (auto &I : *BB) |
| 253 | if (match(&I, m_Intrinsic<Intrinsic::experimental_guard>())) |
| 254 | CurrentList.push_back(cast<IntrinsicInst>(&I)); |
| 255 | |
| 256 | for (auto *II : CurrentList) |
| 257 | Changed |= eliminateGuardViaWidening(II, DFI, GuardsInBlock); |
| 258 | } |
| 259 | |
| 260 | for (auto *II : EliminatedGuards) |
| 261 | if (!WidenedGuards.count(II)) |
| 262 | II->eraseFromParent(); |
| 263 | |
| 264 | return Changed; |
| 265 | } |
| 266 | |
| 267 | bool GuardWideningImpl::eliminateGuardViaWidening( |
| 268 | IntrinsicInst *GuardInst, const df_iterator<DomTreeNode *> &DFSI, |
| 269 | const DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> & |
| 270 | GuardsInBlock) { |
| 271 | IntrinsicInst *BestSoFar = nullptr; |
| 272 | auto BestScoreSoFar = WS_IllegalOrNegative; |
| 273 | auto *GuardInstLoop = LI.getLoopFor(GuardInst->getParent()); |
| 274 | |
| 275 | // In the set of dominating guards, find the one we can merge GuardInst with |
| 276 | // for the most profit. |
| 277 | for (unsigned i = 0, e = DFSI.getPathLength(); i != e; ++i) { |
| 278 | auto *CurBB = DFSI.getPath(i)->getBlock(); |
| 279 | auto *CurLoop = LI.getLoopFor(CurBB); |
| 280 | assert(GuardsInBlock.count(CurBB) && "Must have been populated by now!"); |
| 281 | const auto &GuardsInCurBB = GuardsInBlock.find(CurBB)->second; |
| 282 | |
| 283 | auto I = GuardsInCurBB.begin(); |
| 284 | auto E = GuardsInCurBB.end(); |
| 285 | |
| 286 | #ifndef NDEBUG |
| 287 | { |
| 288 | unsigned Index = 0; |
| 289 | for (auto &I : *CurBB) { |
| 290 | if (Index == GuardsInCurBB.size()) |
| 291 | break; |
| 292 | if (GuardsInCurBB[Index] == &I) |
| 293 | Index++; |
| 294 | } |
| 295 | assert(Index == GuardsInCurBB.size() && |
| 296 | "Guards expected to be in order!"); |
| 297 | } |
| 298 | #endif |
| 299 | |
| 300 | assert((i == (e - 1)) == (GuardInst->getParent() == CurBB) && "Bad DFS?"); |
| 301 | |
| 302 | if (i == (e - 1)) { |
| 303 | // Corner case: make sure we're only looking at guards strictly dominating |
| 304 | // GuardInst when visiting GuardInst->getParent(). |
| 305 | auto NewEnd = std::find(I, E, GuardInst); |
| 306 | assert(NewEnd != E && "GuardInst not in its own block?"); |
| 307 | E = NewEnd; |
| 308 | } |
| 309 | |
| 310 | for (auto *Candidate : make_range(I, E)) { |
| 311 | auto Score = |
| 312 | computeWideningScore(GuardInst, GuardInstLoop, Candidate, CurLoop); |
| 313 | DEBUG(dbgs() << "Score between " << *GuardInst->getArgOperand(0) |
| 314 | << " and " << *Candidate->getArgOperand(0) << " is " |
| 315 | << scoreTypeToString(Score) << "\n"); |
| 316 | if (Score > BestScoreSoFar) { |
| 317 | BestScoreSoFar = Score; |
| 318 | BestSoFar = Candidate; |
| 319 | } |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | if (BestScoreSoFar == WS_IllegalOrNegative) { |
| 324 | DEBUG(dbgs() << "Did not eliminate guard " << *GuardInst << "\n"); |
| 325 | return false; |
| 326 | } |
| 327 | |
| 328 | assert(BestSoFar != GuardInst && "Should have never visited same guard!"); |
| 329 | assert(DT.dominates(BestSoFar, GuardInst) && "Should be!"); |
| 330 | |
| 331 | DEBUG(dbgs() << "Widening " << *GuardInst << " into " << *BestSoFar |
| 332 | << " with score " << scoreTypeToString(BestScoreSoFar) << "\n"); |
| 333 | widenGuard(BestSoFar, GuardInst->getArgOperand(0)); |
| 334 | GuardInst->setArgOperand(0, ConstantInt::getTrue(GuardInst->getContext())); |
| 335 | EliminatedGuards.push_back(GuardInst); |
| 336 | WidenedGuards.insert(BestSoFar); |
| 337 | return true; |
| 338 | } |
| 339 | |
| 340 | GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore( |
| 341 | IntrinsicInst *DominatedGuard, Loop *DominatedGuardLoop, |
| 342 | IntrinsicInst *DominatingGuard, Loop *DominatingGuardLoop) { |
| 343 | bool HoistingOutOfLoop = false; |
| 344 | |
| 345 | if (DominatingGuardLoop != DominatedGuardLoop) { |
| 346 | if (DominatingGuardLoop && |
| 347 | !DominatingGuardLoop->contains(DominatedGuardLoop)) |
| 348 | return WS_IllegalOrNegative; |
| 349 | |
| 350 | HoistingOutOfLoop = true; |
| 351 | } |
| 352 | |
| 353 | if (!isAvailableAt(DominatedGuard->getArgOperand(0), DominatingGuard)) |
| 354 | return WS_IllegalOrNegative; |
| 355 | |
| 356 | bool HoistingOutOfIf = |
| 357 | !PDT.dominates(DominatedGuard->getParent(), DominatingGuard->getParent()); |
| 358 | |
| 359 | if (isWideningCondProfitable(DominatedGuard->getArgOperand(0), |
| 360 | DominatingGuard->getArgOperand(0))) |
| 361 | return HoistingOutOfLoop ? WS_VeryPositive : WS_Positive; |
| 362 | |
| 363 | if (HoistingOutOfLoop) |
| 364 | return WS_Positive; |
| 365 | |
| 366 | return HoistingOutOfIf ? WS_IllegalOrNegative : WS_Neutral; |
| 367 | } |
| 368 | |
| 369 | bool GuardWideningImpl::isAvailableAt(Value *V, Instruction *Loc, |
| 370 | SmallPtrSetImpl<Instruction *> &Visited) { |
| 371 | auto *Inst = dyn_cast<Instruction>(V); |
| 372 | if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst)) |
| 373 | return true; |
| 374 | |
| 375 | if (!isSafeToSpeculativelyExecute(Inst, Loc, &DT) || |
| 376 | Inst->mayReadFromMemory()) |
| 377 | return false; |
| 378 | |
| 379 | Visited.insert(Inst); |
| 380 | |
| 381 | // We only want to go _up_ the dominance chain when recursing. |
| 382 | assert(!isa<PHINode>(Loc) && |
| 383 | "PHIs should return false for isSafeToSpeculativelyExecute"); |
| 384 | assert(DT.isReachableFromEntry(Inst->getParent()) && |
| 385 | "We did a DFS from the block entry!"); |
| 386 | return all_of(Inst->operands(), |
| 387 | [&](Value *Op) { return isAvailableAt(Op, Loc, Visited); }); |
| 388 | } |
| 389 | |
| 390 | void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) { |
| 391 | auto *Inst = dyn_cast<Instruction>(V); |
| 392 | if (!Inst || DT.dominates(Inst, Loc)) |
| 393 | return; |
| 394 | |
| 395 | assert(isSafeToSpeculativelyExecute(Inst, Loc, &DT) && |
| 396 | !Inst->mayReadFromMemory() && "Should've checked with isAvailableAt!"); |
| 397 | |
| 398 | for (Value *Op : Inst->operands()) |
| 399 | makeAvailableAt(Op, Loc); |
| 400 | |
| 401 | Inst->moveBefore(Loc); |
| 402 | } |
| 403 | |
| 404 | bool GuardWideningImpl::widenCondCommon(Value *Cond0, Value *Cond1, |
| 405 | Instruction *InsertPt, Value *&Result) { |
| 406 | using namespace llvm::PatternMatch; |
| 407 | |
| 408 | { |
| 409 | // L >u C0 && L >u C1 -> L >u max(C0, C1) |
| 410 | ConstantInt *RHS0, *RHS1; |
| 411 | Value *LHS; |
| 412 | ICmpInst::Predicate Pred0, Pred1; |
| 413 | if (match(Cond0, m_ICmp(Pred0, m_Value(LHS), m_ConstantInt(RHS0))) && |
| 414 | match(Cond1, m_ICmp(Pred1, m_Specific(LHS), m_ConstantInt(RHS1)))) { |
| 415 | |
Sanjoy Das | b784ed3 | 2016-05-19 03:53:17 +0000 | [diff] [blame] | 416 | ConstantRange CR0 = |
| 417 | ConstantRange::makeExactICmpRegion(Pred0, RHS0->getValue()); |
| 418 | ConstantRange CR1 = |
| 419 | ConstantRange::makeExactICmpRegion(Pred1, RHS1->getValue()); |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 420 | |
Sanjoy Das | b784ed3 | 2016-05-19 03:53:17 +0000 | [diff] [blame] | 421 | // SubsetIntersect is a subset of the actual mathematical intersection of |
Sanjay Patel | f8ee0e0 | 2016-06-19 17:20:27 +0000 | [diff] [blame] | 422 | // CR0 and CR1, while SupersetIntersect is a superset of the actual |
Sanjoy Das | b784ed3 | 2016-05-19 03:53:17 +0000 | [diff] [blame] | 423 | // mathematical intersection. If these two ConstantRanges are equal, then |
| 424 | // we know we were able to represent the actual mathematical intersection |
| 425 | // of CR0 and CR1, and can use the same to generate an icmp instruction. |
| 426 | // |
| 427 | // Given what we're doing here and the semantics of guards, it would |
| 428 | // actually be correct to just use SubsetIntersect, but that may be too |
| 429 | // aggressive in cases we care about. |
| 430 | auto SubsetIntersect = CR0.inverse().unionWith(CR1.inverse()).inverse(); |
| 431 | auto SupersetIntersect = CR0.intersectWith(CR1); |
| 432 | |
| 433 | APInt NewRHSAP; |
| 434 | CmpInst::Predicate Pred; |
| 435 | if (SubsetIntersect == SupersetIntersect && |
| 436 | SubsetIntersect.getEquivalentICmp(Pred, NewRHSAP)) { |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 437 | if (InsertPt) { |
Sanjoy Das | b784ed3 | 2016-05-19 03:53:17 +0000 | [diff] [blame] | 438 | ConstantInt *NewRHS = ConstantInt::get(Cond0->getContext(), NewRHSAP); |
| 439 | Result = new ICmpInst(InsertPt, Pred, LHS, NewRHS, "wide.chk"); |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 440 | } |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 441 | return true; |
| 442 | } |
| 443 | } |
| 444 | } |
| 445 | |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 446 | { |
| 447 | SmallVector<GuardWideningImpl::RangeCheck, 4> Checks, CombinedChecks; |
| 448 | if (parseRangeChecks(Cond0, Checks) && parseRangeChecks(Cond1, Checks) && |
| 449 | combineRangeChecks(Checks, CombinedChecks)) { |
| 450 | if (InsertPt) { |
| 451 | Result = nullptr; |
| 452 | for (auto &RC : CombinedChecks) { |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 453 | makeAvailableAt(RC.getCheckInst(), InsertPt); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 454 | if (Result) |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 455 | Result = BinaryOperator::CreateAnd(RC.getCheckInst(), Result, "", |
| 456 | InsertPt); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 457 | else |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 458 | Result = RC.getCheckInst(); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 459 | } |
| 460 | |
| 461 | Result->setName("wide.chk"); |
| 462 | } |
| 463 | return true; |
| 464 | } |
| 465 | } |
| 466 | |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 467 | // Base case -- just logical-and the two conditions together. |
| 468 | |
| 469 | if (InsertPt) { |
| 470 | makeAvailableAt(Cond0, InsertPt); |
| 471 | makeAvailableAt(Cond1, InsertPt); |
| 472 | |
| 473 | Result = BinaryOperator::CreateAnd(Cond0, Cond1, "wide.chk", InsertPt); |
| 474 | } |
| 475 | |
| 476 | // We were not able to compute Cond0 AND Cond1 for the price of one. |
| 477 | return false; |
| 478 | } |
| 479 | |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 480 | bool GuardWideningImpl::parseRangeChecks( |
| 481 | Value *CheckCond, SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks, |
| 482 | SmallPtrSetImpl<Value *> &Visited) { |
| 483 | if (!Visited.insert(CheckCond).second) |
| 484 | return true; |
| 485 | |
| 486 | using namespace llvm::PatternMatch; |
| 487 | |
| 488 | { |
| 489 | Value *AndLHS, *AndRHS; |
| 490 | if (match(CheckCond, m_And(m_Value(AndLHS), m_Value(AndRHS)))) |
| 491 | return parseRangeChecks(AndLHS, Checks) && |
| 492 | parseRangeChecks(AndRHS, Checks); |
| 493 | } |
| 494 | |
| 495 | auto *IC = dyn_cast<ICmpInst>(CheckCond); |
| 496 | if (!IC || !IC->getOperand(0)->getType()->isIntegerTy() || |
| 497 | (IC->getPredicate() != ICmpInst::ICMP_ULT && |
| 498 | IC->getPredicate() != ICmpInst::ICMP_UGT)) |
| 499 | return false; |
| 500 | |
| 501 | Value *CmpLHS = IC->getOperand(0), *CmpRHS = IC->getOperand(1); |
| 502 | if (IC->getPredicate() == ICmpInst::ICMP_UGT) |
| 503 | std::swap(CmpLHS, CmpRHS); |
| 504 | |
| 505 | auto &DL = IC->getModule()->getDataLayout(); |
| 506 | |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 507 | GuardWideningImpl::RangeCheck Check( |
| 508 | CmpLHS, cast<ConstantInt>(ConstantInt::getNullValue(CmpRHS->getType())), |
| 509 | CmpRHS, IC); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 510 | |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 511 | if (!isKnownNonNegative(Check.getLength(), DL)) |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 512 | return false; |
| 513 | |
| 514 | // What we have in \c Check now is a correct interpretation of \p CheckCond. |
| 515 | // Try to see if we can move some constant offsets into the \c Offset field. |
| 516 | |
| 517 | bool Changed; |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 518 | auto &Ctx = CheckCond->getContext(); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 519 | |
| 520 | do { |
| 521 | Value *OpLHS; |
| 522 | ConstantInt *OpRHS; |
| 523 | Changed = false; |
| 524 | |
| 525 | #ifndef NDEBUG |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 526 | auto *BaseInst = dyn_cast<Instruction>(Check.getBase()); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 527 | assert((!BaseInst || DT.isReachableFromEntry(BaseInst->getParent())) && |
| 528 | "Unreachable instruction?"); |
| 529 | #endif |
| 530 | |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 531 | if (match(Check.getBase(), m_Add(m_Value(OpLHS), m_ConstantInt(OpRHS)))) { |
| 532 | Check.setBase(OpLHS); |
| 533 | APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue(); |
| 534 | Check.setOffset(ConstantInt::get(Ctx, NewOffset)); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 535 | Changed = true; |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 536 | } else if (match(Check.getBase(), |
| 537 | m_Or(m_Value(OpLHS), m_ConstantInt(OpRHS)))) { |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 538 | unsigned BitWidth = OpLHS->getType()->getScalarSizeInBits(); |
| 539 | APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0); |
| 540 | computeKnownBits(OpLHS, KnownZero, KnownOne, DL); |
| 541 | if ((OpRHS->getValue() & KnownZero) == OpRHS->getValue()) { |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 542 | Check.setBase(OpLHS); |
| 543 | APInt NewOffset = Check.getOffsetValue() + OpRHS->getValue(); |
| 544 | Check.setOffset(ConstantInt::get(Ctx, NewOffset)); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 545 | Changed = true; |
| 546 | } |
| 547 | } |
| 548 | } while (Changed); |
| 549 | |
| 550 | Checks.push_back(Check); |
| 551 | return true; |
| 552 | } |
| 553 | |
| 554 | bool GuardWideningImpl::combineRangeChecks( |
| 555 | SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks, |
| 556 | SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) { |
| 557 | unsigned OldCount = Checks.size(); |
| 558 | while (!Checks.empty()) { |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 559 | // Pick all of the range checks with a specific base and length, and try to |
| 560 | // merge them. |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 561 | Value *CurrentBase = Checks.front().getBase(); |
| 562 | Value *CurrentLength = Checks.front().getLength(); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 563 | |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 564 | SmallVector<GuardWideningImpl::RangeCheck, 3> CurrentChecks; |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 565 | |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 566 | auto IsCurrentCheck = [&](GuardWideningImpl::RangeCheck &RC) { |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 567 | return RC.getBase() == CurrentBase && RC.getLength() == CurrentLength; |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 568 | }; |
| 569 | |
| 570 | std::copy_if(Checks.begin(), Checks.end(), |
| 571 | std::back_inserter(CurrentChecks), IsCurrentCheck); |
| 572 | Checks.erase(remove_if(Checks, IsCurrentCheck), Checks.end()); |
| 573 | |
| 574 | assert(CurrentChecks.size() != 0 && "We know we have at least one!"); |
| 575 | |
| 576 | if (CurrentChecks.size() < 3) { |
| 577 | RangeChecksOut.insert(RangeChecksOut.end(), CurrentChecks.begin(), |
| 578 | CurrentChecks.end()); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 579 | continue; |
| 580 | } |
| 581 | |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 582 | // CurrentChecks.size() will typically be 3 here, but so far there has been |
| 583 | // no need to hard-code that fact. |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 584 | |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 585 | std::sort(CurrentChecks.begin(), CurrentChecks.end(), |
Sanjoy Das | 2351975 | 2016-05-19 23:15:59 +0000 | [diff] [blame] | 586 | [&](const GuardWideningImpl::RangeCheck &LHS, |
| 587 | const GuardWideningImpl::RangeCheck &RHS) { |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 588 | return LHS.getOffsetValue().slt(RHS.getOffsetValue()); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 589 | }); |
| 590 | |
| 591 | // Note: std::sort should not invalidate the ChecksStart iterator. |
| 592 | |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 593 | ConstantInt *MinOffset = CurrentChecks.front().getOffset(), |
| 594 | *MaxOffset = CurrentChecks.back().getOffset(); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 595 | |
| 596 | unsigned BitWidth = MaxOffset->getValue().getBitWidth(); |
| 597 | if ((MaxOffset->getValue() - MinOffset->getValue()) |
| 598 | .ugt(APInt::getSignedMinValue(BitWidth))) |
| 599 | return false; |
| 600 | |
| 601 | APInt MaxDiff = MaxOffset->getValue() - MinOffset->getValue(); |
Benjamin Kramer | 46e38f3 | 2016-06-08 10:01:20 +0000 | [diff] [blame] | 602 | const APInt &HighOffset = MaxOffset->getValue(); |
Sanjoy Das | 2351975 | 2016-05-19 23:15:59 +0000 | [diff] [blame] | 603 | auto OffsetOK = [&](const GuardWideningImpl::RangeCheck &RC) { |
Sanjoy Das | be99153 | 2016-05-24 20:54:45 +0000 | [diff] [blame] | 604 | return (HighOffset - RC.getOffsetValue()).ult(MaxDiff); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 605 | }; |
| 606 | |
| 607 | if (MaxDiff.isMinValue() || |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 608 | !std::all_of(std::next(CurrentChecks.begin()), CurrentChecks.end(), |
| 609 | OffsetOK)) |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 610 | return false; |
| 611 | |
| 612 | // We have a series of f+1 checks as: |
| 613 | // |
| 614 | // I+k_0 u< L ... Chk_0 |
| 615 | // I_k_1 u< L ... Chk_1 |
| 616 | // ... |
| 617 | // I_k_f u< L ... Chk_(f+1) |
| 618 | // |
| 619 | // with forall i in [0,f): k_f-k_i u< k_f-k_0 ... Precond_0 |
| 620 | // k_f-k_0 u< INT_MIN+k_f ... Precond_1 |
| 621 | // k_f != k_0 ... Precond_2 |
| 622 | // |
| 623 | // Claim: |
| 624 | // Chk_0 AND Chk_(f+1) implies all the other checks |
| 625 | // |
| 626 | // Informal proof sketch: |
| 627 | // |
| 628 | // We will show that the integer range [I+k_0,I+k_f] does not unsigned-wrap |
| 629 | // (i.e. going from I+k_0 to I+k_f does not cross the -1,0 boundary) and |
| 630 | // thus I+k_f is the greatest unsigned value in that range. |
| 631 | // |
| 632 | // This combined with Ckh_(f+1) shows that everything in that range is u< L. |
| 633 | // Via Precond_0 we know that all of the indices in Chk_0 through Chk_(f+1) |
| 634 | // lie in [I+k_0,I+k_f], this proving our claim. |
| 635 | // |
| 636 | // To see that [I+k_0,I+k_f] is not a wrapping range, note that there are |
| 637 | // two possibilities: I+k_0 u< I+k_f or I+k_0 >u I+k_f (they can't be equal |
| 638 | // since k_0 != k_f). In the former case, [I+k_0,I+k_f] is not a wrapping |
| 639 | // range by definition, and the latter case is impossible: |
| 640 | // |
| 641 | // 0-----I+k_f---I+k_0----L---INT_MAX,INT_MIN------------------(-1) |
| 642 | // xxxxxx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx |
| 643 | // |
| 644 | // For Chk_0 to succeed, we'd have to have k_f-k_0 (the range highlighted |
| 645 | // with 'x' above) to be at least >u INT_MIN. |
| 646 | |
Sanjoy Das | be6c7a1 | 2016-05-21 02:24:44 +0000 | [diff] [blame] | 647 | RangeChecksOut.emplace_back(CurrentChecks.front()); |
| 648 | RangeChecksOut.emplace_back(CurrentChecks.back()); |
Sanjoy Das | f5f0331 | 2016-05-19 22:55:46 +0000 | [diff] [blame] | 649 | } |
| 650 | |
| 651 | assert(RangeChecksOut.size() <= OldCount && "We pessimized!"); |
| 652 | return RangeChecksOut.size() != OldCount; |
| 653 | } |
| 654 | |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 655 | PreservedAnalyses GuardWideningPass::run(Function &F, |
Sean Silva | 36e0d01 | 2016-08-09 00:28:15 +0000 | [diff] [blame^] | 656 | FunctionAnalysisManager &AM) { |
Sanjoy Das | 083f389 | 2016-05-18 22:55:34 +0000 | [diff] [blame] | 657 | auto &DT = AM.getResult<DominatorTreeAnalysis>(F); |
| 658 | auto &LI = AM.getResult<LoopAnalysis>(F); |
| 659 | auto &PDT = AM.getResult<PostDominatorTreeAnalysis>(F); |
| 660 | bool Changed = GuardWideningImpl(DT, PDT, LI).run(); |
| 661 | return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); |
| 662 | } |
| 663 | |
| 664 | StringRef GuardWideningImpl::scoreTypeToString(WideningScore WS) { |
| 665 | switch (WS) { |
| 666 | case WS_IllegalOrNegative: |
| 667 | return "IllegalOrNegative"; |
| 668 | case WS_Neutral: |
| 669 | return "Neutral"; |
| 670 | case WS_Positive: |
| 671 | return "Positive"; |
| 672 | case WS_VeryPositive: |
| 673 | return "VeryPositive"; |
| 674 | } |
| 675 | |
| 676 | llvm_unreachable("Fully covered switch above!"); |
| 677 | } |
| 678 | |
| 679 | char GuardWideningLegacyPass::ID = 0; |
| 680 | |
| 681 | INITIALIZE_PASS_BEGIN(GuardWideningLegacyPass, "guard-widening", "Widen guards", |
| 682 | false, false) |
| 683 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) |
| 684 | INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass) |
| 685 | INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) |
| 686 | INITIALIZE_PASS_END(GuardWideningLegacyPass, "guard-widening", "Widen guards", |
| 687 | false, false) |
| 688 | |
| 689 | FunctionPass *llvm::createGuardWideningPass() { |
| 690 | return new GuardWideningLegacyPass(); |
| 691 | } |