blob: 59e9d2eb928c946d0540c935bb8d111f9b3d353a [file] [log] [blame]
Chris Lattner965c7692008-06-02 01:18:21 +00001//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Chris Lattner965c7692008-06-02 01:18:21 +00006//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000015#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/None.h"
James Molloy493e57d2015-10-26 14:10:46 +000019#include "llvm/ADT/Optional.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000020#include "llvm/ADT/STLExtras.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000021#include "llvm/ADT/SmallPtrSet.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000022#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/ADT/iterator_range.h"
26#include "llvm/Analysis/AliasAnalysis.h"
Daniel Jasperaec2fa32016-12-19 08:22:17 +000027#include "llvm/Analysis/AssumptionCache.h"
Max Kazantsev3c284bd2018-08-30 03:39:16 +000028#include "llvm/Analysis/GuardUtils.h"
Dan Gohman949ab782010-12-15 20:10:26 +000029#include "llvm/Analysis/InstructionSimplify.h"
Artur Pilipenko31bcca42016-02-24 12:49:04 +000030#include "llvm/Analysis/Loads.h"
Adam Nemete2b885c2015-04-23 20:09:20 +000031#include "llvm/Analysis/LoopInfo.h"
Adam Nemet0965da22017-10-09 23:19:02 +000032#include "llvm/Analysis/OptimizationRemarkEmitter.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000033#include "llvm/Analysis/TargetLibraryInfo.h"
34#include "llvm/IR/Argument.h"
35#include "llvm/IR/Attributes.h"
36#include "llvm/IR/BasicBlock.h"
Nick Lewyckyec373542014-05-20 05:13:21 +000037#include "llvm/IR/CallSite.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000038#include "llvm/IR/Constant.h"
Chandler Carruth8cd041e2014-03-04 12:24:34 +000039#include "llvm/IR/ConstantRange.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000040#include "llvm/IR/Constants.h"
Matthias Braun50ec0b52017-05-19 22:37:09 +000041#include "llvm/IR/DerivedTypes.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000042#include "llvm/IR/DiagnosticInfo.h"
Hal Finkel60db0582014-09-07 18:57:58 +000043#include "llvm/IR/Dominators.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000044#include "llvm/IR/Function.h"
Chandler Carruth03eb0de2014-03-04 10:40:04 +000045#include "llvm/IR/GetElementPtrTypeIterator.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000046#include "llvm/IR/GlobalAlias.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000047#include "llvm/IR/GlobalValue.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000048#include "llvm/IR/GlobalVariable.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000049#include "llvm/IR/InstrTypes.h"
50#include "llvm/IR/Instruction.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000051#include "llvm/IR/Instructions.h"
52#include "llvm/IR/IntrinsicInst.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000053#include "llvm/IR/Intrinsics.h"
Reid Kleckner5d986952019-12-11 07:55:26 -080054#include "llvm/IR/IntrinsicsAArch64.h"
55#include "llvm/IR/IntrinsicsX86.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000056#include "llvm/IR/LLVMContext.h"
57#include "llvm/IR/Metadata.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000058#include "llvm/IR/Module.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000059#include "llvm/IR/Operator.h"
Chandler Carruth820a9082014-03-04 11:08:18 +000060#include "llvm/IR/PatternMatch.h"
Eugene Zelenko75075ef2017-09-01 21:37:29 +000061#include "llvm/IR/Type.h"
62#include "llvm/IR/User.h"
63#include "llvm/IR/Value.h"
64#include "llvm/Support/Casting.h"
65#include "llvm/Support/CommandLine.h"
66#include "llvm/Support/Compiler.h"
67#include "llvm/Support/ErrorHandling.h"
Craig Topperb45eabc2017-04-26 16:39:58 +000068#include "llvm/Support/KnownBits.h"
Chris Lattner965c7692008-06-02 01:18:21 +000069#include "llvm/Support/MathExtras.h"
Matthias Braun37e5d792016-01-28 06:29:33 +000070#include <algorithm>
71#include <array>
Eugene Zelenko75075ef2017-09-01 21:37:29 +000072#include <cassert>
73#include <cstdint>
74#include <iterator>
Fangrui Songf78650a2018-07-30 19:41:25 +000075#include <utility>
Eugene Zelenko75075ef2017-09-01 21:37:29 +000076
Chris Lattner965c7692008-06-02 01:18:21 +000077using namespace llvm;
Duncan Sandsd3951082011-01-25 09:38:29 +000078using namespace llvm::PatternMatch;
79
80const unsigned MaxDepth = 6;
81
Philip Reames1c292272015-03-10 22:43:20 +000082// Controls the number of uses of the value searched for possible
83// dominating comparisons.
84static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
Igor Laevskycea9ede2015-09-29 14:57:52 +000085 cl::Hidden, cl::init(20));
Philip Reames1c292272015-03-10 22:43:20 +000086
Craig Topper6b3940a2017-05-03 22:25:19 +000087/// Returns the bitwidth of the given scalar or pointer type. For vector types,
88/// returns the element type's bitwidth.
Mehdi Aminia28d91d2015-03-10 02:37:25 +000089static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
Duncan Sandsd3951082011-01-25 09:38:29 +000090 if (unsigned BitWidth = Ty->getScalarSizeInBits())
91 return BitWidth;
Matt Arsenaultf55e5e72013-08-10 17:34:08 +000092
Nicola Zaghenf798eb22019-12-12 10:25:14 +000093 return DL.getIndexTypeSizeInBits(Ty);
Duncan Sandsd3951082011-01-25 09:38:29 +000094}
Chris Lattner965c7692008-06-02 01:18:21 +000095
Benjamin Kramercfd8d902014-09-12 08:56:53 +000096namespace {
Eugene Zelenko75075ef2017-09-01 21:37:29 +000097
Hal Finkel60db0582014-09-07 18:57:58 +000098// Simplifying using an assume can only be done in a particular control-flow
99// context (the context instruction provides that context). If an assume and
100// the context instruction are not in the same block then the DT helps in
101// figuring out if we can use it.
102struct Query {
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000103 const DataLayout &DL;
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000104 AssumptionCache *AC;
Hal Finkel60db0582014-09-07 18:57:58 +0000105 const Instruction *CxtI;
106 const DominatorTree *DT;
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000107
Sanjay Patel54656ca2017-02-06 18:26:06 +0000108 // Unlike the other analyses, this may be a nullptr because not all clients
109 // provide it currently.
110 OptimizationRemarkEmitter *ORE;
Hal Finkel60db0582014-09-07 18:57:58 +0000111
Matthias Braun37e5d792016-01-28 06:29:33 +0000112 /// Set of assumptions that should be excluded from further queries.
113 /// This is because of the potential for mutual recursion to cause
114 /// computeKnownBits to repeatedly visit the same assume intrinsic. The
115 /// classic case of this is assume(x = y), which will attempt to determine
116 /// bits in x from bits in y, which will attempt to determine bits in y from
117 /// bits in x, etc. Regarding the mutual recursion, computeKnownBits can call
Craig Topper6e11a052017-05-08 16:22:48 +0000118 /// isKnownNonZero, which calls computeKnownBits and isKnownToBeAPowerOfTwo
119 /// (all of which can call computeKnownBits), and so on.
Li Huang755f75f2016-10-15 19:00:04 +0000120 std::array<const Value *, MaxDepth> Excluded;
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000121
Florian Hahn19f9e322018-08-17 14:39:04 +0000122 /// If true, it is safe to use metadata during simplification.
123 InstrInfoQuery IIQ;
124
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000125 unsigned NumExcluded = 0;
Matthias Braun37e5d792016-01-28 06:29:33 +0000126
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000127 Query(const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI,
Florian Hahn19f9e322018-08-17 14:39:04 +0000128 const DominatorTree *DT, bool UseInstrInfo,
129 OptimizationRemarkEmitter *ORE = nullptr)
130 : DL(DL), AC(AC), CxtI(CxtI), DT(DT), ORE(ORE), IIQ(UseInstrInfo) {}
Hal Finkel60db0582014-09-07 18:57:58 +0000131
132 Query(const Query &Q, const Value *NewExcl)
Florian Hahn19f9e322018-08-17 14:39:04 +0000133 : DL(Q.DL), AC(Q.AC), CxtI(Q.CxtI), DT(Q.DT), ORE(Q.ORE), IIQ(Q.IIQ),
Sanjay Patel54656ca2017-02-06 18:26:06 +0000134 NumExcluded(Q.NumExcluded) {
Matthias Braun37e5d792016-01-28 06:29:33 +0000135 Excluded = Q.Excluded;
136 Excluded[NumExcluded++] = NewExcl;
137 assert(NumExcluded <= Excluded.size());
138 }
139
140 bool isExcluded(const Value *Value) const {
141 if (NumExcluded == 0)
142 return false;
143 auto End = Excluded.begin() + NumExcluded;
144 return std::find(Excluded.begin(), End, Value) != End;
Hal Finkel60db0582014-09-07 18:57:58 +0000145 }
146};
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000147
Benjamin Kramercfd8d902014-09-12 08:56:53 +0000148} // end anonymous namespace
Hal Finkel60db0582014-09-07 18:57:58 +0000149
Sanjay Patel547e9752014-11-04 16:09:50 +0000150// Given the provided Value and, potentially, a context instruction, return
Hal Finkel60db0582014-09-07 18:57:58 +0000151// the preferred context instruction (if any).
152static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) {
153 // If we've been provided with a context instruction, then use that (provided
154 // it has been inserted).
155 if (CxtI && CxtI->getParent())
156 return CxtI;
157
158 // If the value is really an already-inserted instruction, then use that.
159 CxtI = dyn_cast<Instruction>(V);
160 if (CxtI && CxtI->getParent())
161 return CxtI;
162
163 return nullptr;
164}
165
Craig Topperb45eabc2017-04-26 16:39:58 +0000166static void computeKnownBits(const Value *V, KnownBits &Known,
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000167 unsigned Depth, const Query &Q);
Hal Finkel60db0582014-09-07 18:57:58 +0000168
Craig Topperb45eabc2017-04-26 16:39:58 +0000169void llvm::computeKnownBits(const Value *V, KnownBits &Known,
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000170 const DataLayout &DL, unsigned Depth,
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000171 AssumptionCache *AC, const Instruction *CxtI,
Sanjay Patel54656ca2017-02-06 18:26:06 +0000172 const DominatorTree *DT,
Florian Hahn19f9e322018-08-17 14:39:04 +0000173 OptimizationRemarkEmitter *ORE, bool UseInstrInfo) {
Craig Topperb45eabc2017-04-26 16:39:58 +0000174 ::computeKnownBits(V, Known, Depth,
Florian Hahn19f9e322018-08-17 14:39:04 +0000175 Query(DL, AC, safeCxtI(V, CxtI), DT, UseInstrInfo, ORE));
Hal Finkel60db0582014-09-07 18:57:58 +0000176}
177
Craig Topper6e11a052017-05-08 16:22:48 +0000178static KnownBits computeKnownBits(const Value *V, unsigned Depth,
179 const Query &Q);
180
181KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
182 unsigned Depth, AssumptionCache *AC,
183 const Instruction *CxtI,
Craig Toppera2025ea2017-05-24 16:53:03 +0000184 const DominatorTree *DT,
Florian Hahn19f9e322018-08-17 14:39:04 +0000185 OptimizationRemarkEmitter *ORE,
186 bool UseInstrInfo) {
187 return ::computeKnownBits(
188 V, Depth, Query(DL, AC, safeCxtI(V, CxtI), DT, UseInstrInfo, ORE));
Craig Topper6e11a052017-05-08 16:22:48 +0000189}
190
Pete Cooper35b00d52016-08-13 01:05:32 +0000191bool llvm::haveNoCommonBitsSet(const Value *LHS, const Value *RHS,
Florian Hahn19f9e322018-08-17 14:39:04 +0000192 const DataLayout &DL, AssumptionCache *AC,
193 const Instruction *CxtI, const DominatorTree *DT,
194 bool UseInstrInfo) {
Jingyue Wuca321902015-05-14 23:53:19 +0000195 assert(LHS->getType() == RHS->getType() &&
196 "LHS and RHS should have the same type");
197 assert(LHS->getType()->isIntOrIntVectorTy() &&
198 "LHS and RHS should be integers");
Roman Lebedev620b3da2018-04-15 18:59:33 +0000199 // Look for an inverted mask: (X & ~M) op (Y & M).
200 Value *M;
201 if (match(LHS, m_c_And(m_Not(m_Value(M)), m_Value())) &&
202 match(RHS, m_c_And(m_Specific(M), m_Value())))
203 return true;
204 if (match(RHS, m_c_And(m_Not(m_Value(M)), m_Value())) &&
205 match(LHS, m_c_And(m_Specific(M), m_Value())))
206 return true;
Jingyue Wuca321902015-05-14 23:53:19 +0000207 IntegerType *IT = cast<IntegerType>(LHS->getType()->getScalarType());
Craig Topperb45eabc2017-04-26 16:39:58 +0000208 KnownBits LHSKnown(IT->getBitWidth());
209 KnownBits RHSKnown(IT->getBitWidth());
Florian Hahn19f9e322018-08-17 14:39:04 +0000210 computeKnownBits(LHS, LHSKnown, DL, 0, AC, CxtI, DT, nullptr, UseInstrInfo);
211 computeKnownBits(RHS, RHSKnown, DL, 0, AC, CxtI, DT, nullptr, UseInstrInfo);
Craig Topperb45eabc2017-04-26 16:39:58 +0000212 return (LHSKnown.Zero | RHSKnown.Zero).isAllOnesValue();
Jingyue Wuca321902015-05-14 23:53:19 +0000213}
214
Zaara Syeda3a7578c2017-05-31 17:12:38 +0000215bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI) {
216 for (const User *U : CxtI->users()) {
217 if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
218 if (IC->isEquality())
219 if (Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
220 if (C->isNullValue())
221 continue;
222 return false;
223 }
224 return true;
225}
226
Pete Cooper35b00d52016-08-13 01:05:32 +0000227static bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000228 const Query &Q);
Hal Finkel60db0582014-09-07 18:57:58 +0000229
Pete Cooper35b00d52016-08-13 01:05:32 +0000230bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
Florian Hahn19f9e322018-08-17 14:39:04 +0000231 bool OrZero, unsigned Depth,
232 AssumptionCache *AC, const Instruction *CxtI,
233 const DominatorTree *DT, bool UseInstrInfo) {
234 return ::isKnownToBeAPowerOfTwo(
235 V, OrZero, Depth, Query(DL, AC, safeCxtI(V, CxtI), DT, UseInstrInfo));
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000236}
237
Pete Cooper35b00d52016-08-13 01:05:32 +0000238static bool isKnownNonZero(const Value *V, unsigned Depth, const Query &Q);
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000239
Pete Cooper35b00d52016-08-13 01:05:32 +0000240bool llvm::isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth,
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000241 AssumptionCache *AC, const Instruction *CxtI,
Florian Hahn19f9e322018-08-17 14:39:04 +0000242 const DominatorTree *DT, bool UseInstrInfo) {
243 return ::isKnownNonZero(V, Depth,
244 Query(DL, AC, safeCxtI(V, CxtI), DT, UseInstrInfo));
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000245}
246
Pete Cooper35b00d52016-08-13 01:05:32 +0000247bool llvm::isKnownNonNegative(const Value *V, const DataLayout &DL,
Florian Hahn19f9e322018-08-17 14:39:04 +0000248 unsigned Depth, AssumptionCache *AC,
249 const Instruction *CxtI, const DominatorTree *DT,
250 bool UseInstrInfo) {
251 KnownBits Known =
252 computeKnownBits(V, DL, Depth, AC, CxtI, DT, nullptr, UseInstrInfo);
Craig Topper6e11a052017-05-08 16:22:48 +0000253 return Known.isNonNegative();
Jingyue Wu10fcea52015-08-20 18:27:04 +0000254}
255
Pete Cooper35b00d52016-08-13 01:05:32 +0000256bool llvm::isKnownPositive(const Value *V, const DataLayout &DL, unsigned Depth,
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000257 AssumptionCache *AC, const Instruction *CxtI,
Florian Hahn19f9e322018-08-17 14:39:04 +0000258 const DominatorTree *DT, bool UseInstrInfo) {
Philip Reames8f12eba2016-03-09 21:31:47 +0000259 if (auto *CI = dyn_cast<ConstantInt>(V))
260 return CI->getValue().isStrictlyPositive();
Sanjoy Das6082c1a2016-05-07 02:08:15 +0000261
Philip Reames8f12eba2016-03-09 21:31:47 +0000262 // TODO: We'd doing two recursive queries here. We should factor this such
263 // that only a single query is needed.
Florian Hahn19f9e322018-08-17 14:39:04 +0000264 return isKnownNonNegative(V, DL, Depth, AC, CxtI, DT, UseInstrInfo) &&
265 isKnownNonZero(V, DL, Depth, AC, CxtI, DT, UseInstrInfo);
Philip Reames8f12eba2016-03-09 21:31:47 +0000266}
267
Pete Cooper35b00d52016-08-13 01:05:32 +0000268bool llvm::isKnownNegative(const Value *V, const DataLayout &DL, unsigned Depth,
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000269 AssumptionCache *AC, const Instruction *CxtI,
Florian Hahn19f9e322018-08-17 14:39:04 +0000270 const DominatorTree *DT, bool UseInstrInfo) {
271 KnownBits Known =
272 computeKnownBits(V, DL, Depth, AC, CxtI, DT, nullptr, UseInstrInfo);
Craig Topper6e11a052017-05-08 16:22:48 +0000273 return Known.isNegative();
Nick Lewycky762f8a82016-04-21 00:53:14 +0000274}
275
Pete Cooper35b00d52016-08-13 01:05:32 +0000276static bool isKnownNonEqual(const Value *V1, const Value *V2, const Query &Q);
James Molloy1d88d6f2015-10-22 13:18:42 +0000277
Pete Cooper35b00d52016-08-13 01:05:32 +0000278bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
Florian Hahn19f9e322018-08-17 14:39:04 +0000279 const DataLayout &DL, AssumptionCache *AC,
280 const Instruction *CxtI, const DominatorTree *DT,
281 bool UseInstrInfo) {
282 return ::isKnownNonEqual(V1, V2,
283 Query(DL, AC, safeCxtI(V1, safeCxtI(V2, CxtI)), DT,
284 UseInstrInfo, /*ORE=*/nullptr));
James Molloy1d88d6f2015-10-22 13:18:42 +0000285}
286
Pete Cooper35b00d52016-08-13 01:05:32 +0000287static bool MaskedValueIsZero(const Value *V, const APInt &Mask, unsigned Depth,
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000288 const Query &Q);
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000289
Pete Cooper35b00d52016-08-13 01:05:32 +0000290bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
Florian Hahn19f9e322018-08-17 14:39:04 +0000291 const DataLayout &DL, unsigned Depth,
292 AssumptionCache *AC, const Instruction *CxtI,
293 const DominatorTree *DT, bool UseInstrInfo) {
294 return ::MaskedValueIsZero(
295 V, Mask, Depth, Query(DL, AC, safeCxtI(V, CxtI), DT, UseInstrInfo));
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000296}
297
Pete Cooper35b00d52016-08-13 01:05:32 +0000298static unsigned ComputeNumSignBits(const Value *V, unsigned Depth,
299 const Query &Q);
Mehdi Aminia28d91d2015-03-10 02:37:25 +0000300
Pete Cooper35b00d52016-08-13 01:05:32 +0000301unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000302 unsigned Depth, AssumptionCache *AC,
303 const Instruction *CxtI,
Florian Hahn19f9e322018-08-17 14:39:04 +0000304 const DominatorTree *DT, bool UseInstrInfo) {
305 return ::ComputeNumSignBits(
306 V, Depth, Query(DL, AC, safeCxtI(V, CxtI), DT, UseInstrInfo));
Hal Finkel60db0582014-09-07 18:57:58 +0000307}
308
Craig Topper8fbb74b2017-03-24 22:12:10 +0000309static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
310 bool NSW,
Craig Topperb45eabc2017-04-26 16:39:58 +0000311 KnownBits &KnownOut, KnownBits &Known2,
Craig Topper8fbb74b2017-03-24 22:12:10 +0000312 unsigned Depth, const Query &Q) {
Craig Topperb45eabc2017-04-26 16:39:58 +0000313 unsigned BitWidth = KnownOut.getBitWidth();
Craig Topper8fbb74b2017-03-24 22:12:10 +0000314
315 // If an initial sequence of bits in the result is not needed, the
316 // corresponding bits in the operands are not needed.
Craig Topperb45eabc2017-04-26 16:39:58 +0000317 KnownBits LHSKnown(BitWidth);
318 computeKnownBits(Op0, LHSKnown, Depth + 1, Q);
319 computeKnownBits(Op1, Known2, Depth + 1, Q);
Craig Topper8fbb74b2017-03-24 22:12:10 +0000320
Craig Topperb498a232017-08-08 16:29:35 +0000321 KnownOut = KnownBits::computeForAddSub(Add, NSW, LHSKnown, Known2);
Nick Lewyckyfea3e002012-03-09 09:23:50 +0000322}
323
Pete Cooper35b00d52016-08-13 01:05:32 +0000324static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
Craig Topperb45eabc2017-04-26 16:39:58 +0000325 KnownBits &Known, KnownBits &Known2,
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000326 unsigned Depth, const Query &Q) {
Craig Topperb45eabc2017-04-26 16:39:58 +0000327 unsigned BitWidth = Known.getBitWidth();
328 computeKnownBits(Op1, Known, Depth + 1, Q);
329 computeKnownBits(Op0, Known2, Depth + 1, Q);
Nick Lewyckyfa306072012-03-18 23:28:48 +0000330
331 bool isKnownNegative = false;
332 bool isKnownNonNegative = false;
333 // If the multiplication is known not to overflow, compute the sign bit.
Rafael Espindolaba0a6ca2012-04-04 12:51:34 +0000334 if (NSW) {
Nick Lewyckyfa306072012-03-18 23:28:48 +0000335 if (Op0 == Op1) {
336 // The product of a number with itself is non-negative.
337 isKnownNonNegative = true;
338 } else {
Craig Topperca48af32017-04-29 16:43:11 +0000339 bool isKnownNonNegativeOp1 = Known.isNonNegative();
340 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
341 bool isKnownNegativeOp1 = Known.isNegative();
342 bool isKnownNegativeOp0 = Known2.isNegative();
Nick Lewyckyfa306072012-03-18 23:28:48 +0000343 // The product of two numbers with the same sign is non-negative.
344 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
345 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
346 // The product of a negative number and a non-negative number is either
347 // negative or zero.
348 if (!isKnownNonNegative)
349 isKnownNegative = (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000350 isKnownNonZero(Op0, Depth, Q)) ||
Nick Lewyckyfa306072012-03-18 23:28:48 +0000351 (isKnownNegativeOp0 && isKnownNonNegativeOp1 &&
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000352 isKnownNonZero(Op1, Depth, Q));
Nick Lewyckyfa306072012-03-18 23:28:48 +0000353 }
354 }
355
Simon Dardis70dbd5f2017-12-09 23:25:57 +0000356 assert(!Known.hasConflict() && !Known2.hasConflict());
357 // Compute a conservative estimate for high known-0 bits.
Craig Topper8df66c62017-05-12 17:20:30 +0000358 unsigned LeadZ = std::max(Known.countMinLeadingZeros() +
359 Known2.countMinLeadingZeros(),
Nick Lewyckyfa306072012-03-18 23:28:48 +0000360 BitWidth) - BitWidth;
Nick Lewyckyfa306072012-03-18 23:28:48 +0000361 LeadZ = std::min(LeadZ, BitWidth);
Simon Dardis70dbd5f2017-12-09 23:25:57 +0000362
363 // The result of the bottom bits of an integer multiply can be
364 // inferred by looking at the bottom bits of both operands and
365 // multiplying them together.
366 // We can infer at least the minimum number of known trailing bits
367 // of both operands. Depending on number of trailing zeros, we can
368 // infer more bits, because (a*b) <=> ((a/m) * (b/n)) * (m*n) assuming
369 // a and b are divisible by m and n respectively.
370 // We then calculate how many of those bits are inferrable and set
371 // the output. For example, the i8 mul:
372 // a = XXXX1100 (12)
373 // b = XXXX1110 (14)
374 // We know the bottom 3 bits are zero since the first can be divided by
375 // 4 and the second by 2, thus having ((12/4) * (14/2)) * (2*4).
376 // Applying the multiplication to the trimmed arguments gets:
377 // XX11 (3)
378 // X111 (7)
379 // -------
380 // XX11
381 // XX11
382 // XX11
383 // XX11
384 // -------
385 // XXXXX01
386 // Which allows us to infer the 2 LSBs. Since we're multiplying the result
387 // by 8, the bottom 3 bits will be 0, so we can infer a total of 5 bits.
388 // The proof for this can be described as:
389 // Pre: (C1 >= 0) && (C1 < (1 << C5)) && (C2 >= 0) && (C2 < (1 << C6)) &&
390 // (C7 == (1 << (umin(countTrailingZeros(C1), C5) +
391 // umin(countTrailingZeros(C2), C6) +
392 // umin(C5 - umin(countTrailingZeros(C1), C5),
393 // C6 - umin(countTrailingZeros(C2), C6)))) - 1)
394 // %aa = shl i8 %a, C5
395 // %bb = shl i8 %b, C6
396 // %aaa = or i8 %aa, C1
397 // %bbb = or i8 %bb, C2
398 // %mul = mul i8 %aaa, %bbb
399 // %mask = and i8 %mul, C7
400 // =>
401 // %mask = i8 ((C1*C2)&C7)
402 // Where C5, C6 describe the known bits of %a, %b
403 // C1, C2 describe the known bottom bits of %a, %b.
404 // C7 describes the mask of the known bits of the result.
405 APInt Bottom0 = Known.One;
406 APInt Bottom1 = Known2.One;
407
408 // How many times we'd be able to divide each argument by 2 (shr by 1).
409 // This gives us the number of trailing zeros on the multiplication result.
410 unsigned TrailBitsKnown0 = (Known.Zero | Known.One).countTrailingOnes();
411 unsigned TrailBitsKnown1 = (Known2.Zero | Known2.One).countTrailingOnes();
412 unsigned TrailZero0 = Known.countMinTrailingZeros();
413 unsigned TrailZero1 = Known2.countMinTrailingZeros();
414 unsigned TrailZ = TrailZero0 + TrailZero1;
415
416 // Figure out the fewest known-bits operand.
417 unsigned SmallestOperand = std::min(TrailBitsKnown0 - TrailZero0,
418 TrailBitsKnown1 - TrailZero1);
419 unsigned ResultBitsKnown = std::min(SmallestOperand + TrailZ, BitWidth);
420
421 APInt BottomKnown = Bottom0.getLoBits(TrailBitsKnown0) *
422 Bottom1.getLoBits(TrailBitsKnown1);
423
Craig Topperf0aeee02017-05-05 17:36:09 +0000424 Known.resetAll();
Craig Topperb45eabc2017-04-26 16:39:58 +0000425 Known.Zero.setHighBits(LeadZ);
Simon Dardis70dbd5f2017-12-09 23:25:57 +0000426 Known.Zero |= (~BottomKnown).getLoBits(ResultBitsKnown);
427 Known.One |= BottomKnown.getLoBits(ResultBitsKnown);
Nick Lewyckyfa306072012-03-18 23:28:48 +0000428
429 // Only make use of no-wrap flags if we failed to compute the sign bit
430 // directly. This matters if the multiplication always overflows, in
431 // which case we prefer to follow the result of the direct computation,
432 // though as the program is invoking undefined behaviour we can choose
433 // whatever we like here.
Craig Topperca48af32017-04-29 16:43:11 +0000434 if (isKnownNonNegative && !Known.isNegative())
435 Known.makeNonNegative();
436 else if (isKnownNegative && !Known.isNonNegative())
437 Known.makeNegative();
Nick Lewyckyfa306072012-03-18 23:28:48 +0000438}
439
Jingyue Wu37fcb592014-06-19 16:50:16 +0000440void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
Craig Topperf42b23f2017-04-28 06:28:56 +0000441 KnownBits &Known) {
442 unsigned BitWidth = Known.getBitWidth();
Rafael Espindola53190532012-03-30 15:52:11 +0000443 unsigned NumRanges = Ranges.getNumOperands() / 2;
444 assert(NumRanges >= 1);
445
Craig Topperf42b23f2017-04-28 06:28:56 +0000446 Known.Zero.setAllBits();
447 Known.One.setAllBits();
Sanjoy Das1d1929a2015-10-28 03:20:15 +0000448
Rafael Espindola53190532012-03-30 15:52:11 +0000449 for (unsigned i = 0; i < NumRanges; ++i) {
Duncan P. N. Exon Smith5bf8fef2014-12-09 18:38:53 +0000450 ConstantInt *Lower =
451 mdconst::extract<ConstantInt>(Ranges.getOperand(2 * i + 0));
452 ConstantInt *Upper =
453 mdconst::extract<ConstantInt>(Ranges.getOperand(2 * i + 1));
Rafael Espindola53190532012-03-30 15:52:11 +0000454 ConstantRange Range(Lower->getValue(), Upper->getValue());
Rafael Espindola53190532012-03-30 15:52:11 +0000455
Sanjoy Das1d1929a2015-10-28 03:20:15 +0000456 // The first CommonPrefixBits of all values in Range are equal.
457 unsigned CommonPrefixBits =
458 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countLeadingZeros();
459
460 APInt Mask = APInt::getHighBitsSet(BitWidth, CommonPrefixBits);
Craig Topperf42b23f2017-04-28 06:28:56 +0000461 Known.One &= Range.getUnsignedMax() & Mask;
462 Known.Zero &= ~Range.getUnsignedMax() & Mask;
Sanjoy Das1d1929a2015-10-28 03:20:15 +0000463 }
Rafael Espindola53190532012-03-30 15:52:11 +0000464}
Jay Foad5a29c362014-05-15 12:12:55 +0000465
Pete Cooperfa7ae4f2016-08-11 22:23:07 +0000466static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
Hal Finkel60db0582014-09-07 18:57:58 +0000467 SmallVector<const Value *, 16> WorkSet(1, I);
468 SmallPtrSet<const Value *, 32> Visited;
469 SmallPtrSet<const Value *, 16> EphValues;
470
Hal Finkelf2199b22015-10-23 20:37:08 +0000471 // The instruction defining an assumption's condition itself is always
472 // considered ephemeral to that assumption (even if it has other
473 // non-ephemeral users). See r246696's test case for an example.
David Majnemer0a16c222016-08-11 21:15:00 +0000474 if (is_contained(I->operands(), E))
Hal Finkelf2199b22015-10-23 20:37:08 +0000475 return true;
476
Hal Finkel60db0582014-09-07 18:57:58 +0000477 while (!WorkSet.empty()) {
478 const Value *V = WorkSet.pop_back_val();
David Blaikie70573dc2014-11-19 07:49:26 +0000479 if (!Visited.insert(V).second)
Hal Finkel60db0582014-09-07 18:57:58 +0000480 continue;
481
482 // If all uses of this value are ephemeral, then so is this value.
Eugene Zelenko75075ef2017-09-01 21:37:29 +0000483 if (llvm::all_of(V->users(), [&](const User *U) {
484 return EphValues.count(U);
485 })) {
Hal Finkel60db0582014-09-07 18:57:58 +0000486 if (V == E)
487 return true;
488
Hal Finkelb03dd4b2017-08-14 17:11:43 +0000489 if (V == I || isSafeToSpeculativelyExecute(V)) {
490 EphValues.insert(V);
491 if (const User *U = dyn_cast<User>(V))
492 for (User::const_op_iterator J = U->op_begin(), JE = U->op_end();
493 J != JE; ++J)
494 WorkSet.push_back(*J);
495 }
Hal Finkel60db0582014-09-07 18:57:58 +0000496 }
497 }
498
499 return false;
500}
501
502// Is this an intrinsic that cannot be speculated but also cannot trap?
Haicheng Wua4461512017-12-15 14:34:41 +0000503bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
Hal Finkel60db0582014-09-07 18:57:58 +0000504 if (const CallInst *CI = dyn_cast<CallInst>(I))
505 if (Function *F = CI->getCalledFunction())
506 switch (F->getIntrinsicID()) {
507 default: break;
508 // FIXME: This list is repeated from NoTTI::getIntrinsicCost.
509 case Intrinsic::assume:
Dan Gohman2c74fe92017-11-08 21:59:51 +0000510 case Intrinsic::sideeffect:
Hal Finkel60db0582014-09-07 18:57:58 +0000511 case Intrinsic::dbg_declare:
512 case Intrinsic::dbg_value:
Shiva Chen2c864552018-05-09 02:40:45 +0000513 case Intrinsic::dbg_label:
Hal Finkel60db0582014-09-07 18:57:58 +0000514 case Intrinsic::invariant_start:
515 case Intrinsic::invariant_end:
516 case Intrinsic::lifetime_start:
517 case Intrinsic::lifetime_end:
518 case Intrinsic::objectsize:
519 case Intrinsic::ptr_annotation:
520 case Intrinsic::var_annotation:
521 return true;
522 }
523
524 return false;
525}
526
Pete Cooperfa7ae4f2016-08-11 22:23:07 +0000527bool llvm::isValidAssumeForContext(const Instruction *Inv,
528 const Instruction *CxtI,
529 const DominatorTree *DT) {
Hal Finkel60db0582014-09-07 18:57:58 +0000530 // There are two restrictions on the use of an assume:
531 // 1. The assume must dominate the context (or the control flow must
532 // reach the assume whenever it reaches the context).
533 // 2. The context must not be in the assume's set of ephemeral values
534 // (otherwise we will use the assume to prove that the condition
535 // feeding the assume is trivially true, thus causing the removal of
536 // the assume).
537
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000538 if (DT) {
Pete Cooper54a02552016-08-12 01:00:15 +0000539 if (DT->dominates(Inv, CxtI))
Hal Finkel60db0582014-09-07 18:57:58 +0000540 return true;
Pete Cooper54a02552016-08-12 01:00:15 +0000541 } else if (Inv->getParent() == CxtI->getParent()->getSinglePredecessor()) {
542 // We don't have a DT, but this trivially dominates.
543 return true;
Hal Finkel60db0582014-09-07 18:57:58 +0000544 }
545
Pete Cooper54a02552016-08-12 01:00:15 +0000546 // With or without a DT, the only remaining case we will check is if the
547 // instructions are in the same BB. Give up if that is not the case.
548 if (Inv->getParent() != CxtI->getParent())
549 return false;
550
Vedant Kumard3196742018-02-28 19:08:52 +0000551 // If we have a dom tree, then we now know that the assume doesn't dominate
Pete Cooper54a02552016-08-12 01:00:15 +0000552 // the other instruction. If we don't have a dom tree then we can check if
553 // the assume is first in the BB.
554 if (!DT) {
Hal Finkel60db0582014-09-07 18:57:58 +0000555 // Search forward from the assume until we reach the context (or the end
556 // of the block); the common case is that the assume will come first.
Pete Cooperfa7ae4f2016-08-11 22:23:07 +0000557 for (auto I = std::next(BasicBlock::const_iterator(Inv)),
Hal Finkel60db0582014-09-07 18:57:58 +0000558 IE = Inv->getParent()->end(); I != IE; ++I)
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000559 if (&*I == CxtI)
Hal Finkel60db0582014-09-07 18:57:58 +0000560 return true;
Hal Finkel60db0582014-09-07 18:57:58 +0000561 }
562
Nikita Popov2a4f26b2019-08-13 17:15:42 +0000563 // Don't let an assume affect itself - this would cause the problems
564 // `isEphemeralValueOf` is trying to prevent, and it would also make
565 // the loop below go out of bounds.
566 if (Inv == CxtI)
567 return false;
568
Pete Cooper54a02552016-08-12 01:00:15 +0000569 // The context comes first, but they're both in the same block. Make sure
570 // there is nothing in between that might interrupt the control flow.
571 for (BasicBlock::const_iterator I =
572 std::next(BasicBlock::const_iterator(CxtI)), IE(Inv);
573 I != IE; ++I)
Nikita Popov2a4f26b2019-08-13 17:15:42 +0000574 if (!isGuaranteedToTransferExecutionToSuccessor(&*I))
Pete Cooper54a02552016-08-12 01:00:15 +0000575 return false;
576
577 return !isEphemeralValueOf(Inv, CxtI);
Hal Finkel60db0582014-09-07 18:57:58 +0000578}
579
Craig Topperb45eabc2017-04-26 16:39:58 +0000580static void computeKnownBitsFromAssume(const Value *V, KnownBits &Known,
581 unsigned Depth, const Query &Q) {
Hal Finkel60db0582014-09-07 18:57:58 +0000582 // Use of assumptions is context-sensitive. If we don't have a context, we
583 // cannot use them!
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000584 if (!Q.AC || !Q.CxtI)
Hal Finkel60db0582014-09-07 18:57:58 +0000585 return;
586
Craig Topperb45eabc2017-04-26 16:39:58 +0000587 unsigned BitWidth = Known.getBitWidth();
Hal Finkel60db0582014-09-07 18:57:58 +0000588
Hal Finkel8a9a7832017-01-11 13:24:24 +0000589 // Note that the patterns below need to be kept in sync with the code
590 // in AssumptionCache::updateAffectedValues.
591
592 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000593 if (!AssumeVH)
Chandler Carruth66b31302015-01-04 12:03:27 +0000594 continue;
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000595 CallInst *I = cast<CallInst>(AssumeVH);
596 assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
597 "Got assumption for the wrong function!");
598 if (Q.isExcluded(I))
Hal Finkel60db0582014-09-07 18:57:58 +0000599 continue;
600
Vedant Kumard3196742018-02-28 19:08:52 +0000601 // Warning: This loop can end up being somewhat performance sensitive.
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000602 // We're running this loop for once for each value queried resulting in a
603 // runtime of ~O(#assumes * #values).
Philip Reames00d3b272014-11-24 23:44:28 +0000604
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000605 assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume &&
606 "must be an assume intrinsic");
607
608 Value *Arg = I->getArgOperand(0);
609
610 if (Arg == V && isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
Hal Finkel60db0582014-09-07 18:57:58 +0000611 assert(BitWidth == 1 && "assume operand is not i1?");
Craig Topperf0aeee02017-05-05 17:36:09 +0000612 Known.setAllOnes();
Hal Finkel60db0582014-09-07 18:57:58 +0000613 return;
614 }
Sanjay Patel96669962017-01-17 18:15:49 +0000615 if (match(Arg, m_Not(m_Specific(V))) &&
616 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
617 assert(BitWidth == 1 && "assume operand is not i1?");
Craig Topperf0aeee02017-05-05 17:36:09 +0000618 Known.setAllZero();
Sanjay Patel96669962017-01-17 18:15:49 +0000619 return;
620 }
Hal Finkel60db0582014-09-07 18:57:58 +0000621
David Majnemer9b609752014-12-12 23:59:29 +0000622 // The remaining tests are all recursive, so bail out if we hit the limit.
623 if (Depth == MaxDepth)
624 continue;
625
Sander de Smalen0e66db52019-04-10 16:24:48 +0000626 ICmpInst *Cmp = dyn_cast<ICmpInst>(Arg);
627 if (!Cmp)
628 continue;
629
Hal Finkel60db0582014-09-07 18:57:58 +0000630 Value *A, *B;
Sanjay Patel2a707032019-03-03 18:59:33 +0000631 auto m_V = m_CombineOr(m_Specific(V), m_PtrToInt(m_Specific(V)));
Hal Finkel60db0582014-09-07 18:57:58 +0000632
633 CmpInst::Predicate Pred;
Igor Laevskycec8f472017-12-05 12:18:15 +0000634 uint64_t C;
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000635 switch (Cmp->getPredicate()) {
636 default:
637 break;
638 case ICmpInst::ICMP_EQ:
Sander de Smalen0e66db52019-04-10 16:24:48 +0000639 // assume(v = a)
640 if (match(Cmp, m_c_ICmp(Pred, m_V, m_Value(A))) &&
641 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
642 KnownBits RHSKnown(BitWidth);
643 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
644 Known.Zero |= RHSKnown.Zero;
645 Known.One |= RHSKnown.One;
646 // assume(v & b = a)
647 } else if (match(Cmp,
648 m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A))) &&
649 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
650 KnownBits RHSKnown(BitWidth);
651 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
652 KnownBits MaskKnown(BitWidth);
653 computeKnownBits(B, MaskKnown, Depth+1, Query(Q, I));
Hal Finkel60db0582014-09-07 18:57:58 +0000654
Sander de Smalen0e66db52019-04-10 16:24:48 +0000655 // For those bits in the mask that are known to be one, we can propagate
656 // known bits from the RHS to V.
657 Known.Zero |= RHSKnown.Zero & MaskKnown.One;
658 Known.One |= RHSKnown.One & MaskKnown.One;
659 // assume(~(v & b) = a)
660 } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))),
661 m_Value(A))) &&
662 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
663 KnownBits RHSKnown(BitWidth);
664 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
665 KnownBits MaskKnown(BitWidth);
666 computeKnownBits(B, MaskKnown, Depth+1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000667
Sander de Smalen0e66db52019-04-10 16:24:48 +0000668 // For those bits in the mask that are known to be one, we can propagate
669 // inverted known bits from the RHS to V.
670 Known.Zero |= RHSKnown.One & MaskKnown.One;
671 Known.One |= RHSKnown.Zero & MaskKnown.One;
672 // assume(v | b = a)
673 } else if (match(Cmp,
674 m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A))) &&
675 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
676 KnownBits RHSKnown(BitWidth);
677 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
678 KnownBits BKnown(BitWidth);
679 computeKnownBits(B, BKnown, Depth+1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000680
Sander de Smalen0e66db52019-04-10 16:24:48 +0000681 // For those bits in B that are known to be zero, we can propagate known
682 // bits from the RHS to V.
683 Known.Zero |= RHSKnown.Zero & BKnown.Zero;
684 Known.One |= RHSKnown.One & BKnown.Zero;
685 // assume(~(v | b) = a)
686 } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))),
687 m_Value(A))) &&
688 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
689 KnownBits RHSKnown(BitWidth);
690 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
691 KnownBits BKnown(BitWidth);
692 computeKnownBits(B, BKnown, Depth+1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000693
Sander de Smalen0e66db52019-04-10 16:24:48 +0000694 // For those bits in B that are known to be zero, we can propagate
695 // inverted known bits from the RHS to V.
696 Known.Zero |= RHSKnown.One & BKnown.Zero;
697 Known.One |= RHSKnown.Zero & BKnown.Zero;
698 // assume(v ^ b = a)
699 } else if (match(Cmp,
700 m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A))) &&
701 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
702 KnownBits RHSKnown(BitWidth);
703 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
704 KnownBits BKnown(BitWidth);
705 computeKnownBits(B, BKnown, Depth+1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000706
Sander de Smalen0e66db52019-04-10 16:24:48 +0000707 // For those bits in B that are known to be zero, we can propagate known
708 // bits from the RHS to V. For those bits in B that are known to be one,
709 // we can propagate inverted known bits from the RHS to V.
710 Known.Zero |= RHSKnown.Zero & BKnown.Zero;
711 Known.One |= RHSKnown.One & BKnown.Zero;
712 Known.Zero |= RHSKnown.One & BKnown.One;
713 Known.One |= RHSKnown.Zero & BKnown.One;
714 // assume(~(v ^ b) = a)
715 } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))),
716 m_Value(A))) &&
717 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
718 KnownBits RHSKnown(BitWidth);
719 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
720 KnownBits BKnown(BitWidth);
721 computeKnownBits(B, BKnown, Depth+1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000722
Sander de Smalen0e66db52019-04-10 16:24:48 +0000723 // For those bits in B that are known to be zero, we can propagate
724 // inverted known bits from the RHS to V. For those bits in B that are
725 // known to be one, we can propagate known bits from the RHS to V.
726 Known.Zero |= RHSKnown.One & BKnown.Zero;
727 Known.One |= RHSKnown.Zero & BKnown.Zero;
728 Known.Zero |= RHSKnown.Zero & BKnown.One;
729 Known.One |= RHSKnown.One & BKnown.One;
730 // assume(v << c = a)
731 } else if (match(Cmp, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)),
732 m_Value(A))) &&
733 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
734 KnownBits RHSKnown(BitWidth);
735 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
736 // For those bits in RHS that are known, we can propagate them to known
737 // bits in V shifted to the right by C.
738 RHSKnown.Zero.lshrInPlace(C);
739 Known.Zero |= RHSKnown.Zero;
740 RHSKnown.One.lshrInPlace(C);
741 Known.One |= RHSKnown.One;
742 // assume(~(v << c) = a)
743 } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))),
744 m_Value(A))) &&
745 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
746 KnownBits RHSKnown(BitWidth);
747 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
748 // For those bits in RHS that are known, we can propagate them inverted
749 // to known bits in V shifted to the right by C.
750 RHSKnown.One.lshrInPlace(C);
751 Known.Zero |= RHSKnown.One;
752 RHSKnown.Zero.lshrInPlace(C);
753 Known.One |= RHSKnown.Zero;
754 // assume(v >> c = a)
755 } else if (match(Cmp, m_c_ICmp(Pred, m_Shr(m_V, m_ConstantInt(C)),
756 m_Value(A))) &&
757 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
758 KnownBits RHSKnown(BitWidth);
759 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
760 // For those bits in RHS that are known, we can propagate them to known
761 // bits in V shifted to the right by C.
762 Known.Zero |= RHSKnown.Zero << C;
763 Known.One |= RHSKnown.One << C;
764 // assume(~(v >> c) = a)
765 } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shr(m_V, m_ConstantInt(C))),
766 m_Value(A))) &&
767 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
768 KnownBits RHSKnown(BitWidth);
769 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
770 // For those bits in RHS that are known, we can propagate them inverted
771 // to known bits in V shifted to the right by C.
772 Known.Zero |= RHSKnown.One << C;
773 Known.One |= RHSKnown.Zero << C;
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000774 }
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000775 break;
776 case ICmpInst::ICMP_SGE:
Sander de Smalen0e66db52019-04-10 16:24:48 +0000777 // assume(v >=_s c) where c is non-negative
778 if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
779 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
780 KnownBits RHSKnown(BitWidth);
781 computeKnownBits(A, RHSKnown, Depth + 1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000782
Sander de Smalen0e66db52019-04-10 16:24:48 +0000783 if (RHSKnown.isNonNegative()) {
784 // We know that the sign bit is zero.
785 Known.makeNonNegative();
786 }
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000787 }
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000788 break;
789 case ICmpInst::ICMP_SGT:
Sander de Smalen0e66db52019-04-10 16:24:48 +0000790 // assume(v >_s c) where c is at least -1.
791 if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
792 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
793 KnownBits RHSKnown(BitWidth);
794 computeKnownBits(A, RHSKnown, Depth + 1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000795
Sander de Smalen0e66db52019-04-10 16:24:48 +0000796 if (RHSKnown.isAllOnes() || RHSKnown.isNonNegative()) {
797 // We know that the sign bit is zero.
798 Known.makeNonNegative();
799 }
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000800 }
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000801 break;
802 case ICmpInst::ICMP_SLE:
Sander de Smalen0e66db52019-04-10 16:24:48 +0000803 // assume(v <=_s c) where c is negative
804 if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
805 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
806 KnownBits RHSKnown(BitWidth);
807 computeKnownBits(A, RHSKnown, Depth + 1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000808
Sander de Smalen0e66db52019-04-10 16:24:48 +0000809 if (RHSKnown.isNegative()) {
810 // We know that the sign bit is one.
811 Known.makeNegative();
812 }
813 }
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000814 break;
815 case ICmpInst::ICMP_SLT:
Sander de Smalen0e66db52019-04-10 16:24:48 +0000816 // assume(v <_s c) where c is non-positive
817 if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
818 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
819 KnownBits RHSKnown(BitWidth);
820 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
821
822 if (RHSKnown.isZero() || RHSKnown.isNegative()) {
823 // We know that the sign bit is one.
824 Known.makeNegative();
825 }
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000826 }
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000827 break;
828 case ICmpInst::ICMP_ULE:
829 // assume(v <=_u c)
Sander de Smalen0e66db52019-04-10 16:24:48 +0000830 if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
831 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
832 KnownBits RHSKnown(BitWidth);
833 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
Hal Finkel15aeaaf2014-09-07 19:21:07 +0000834
Sander de Smalen0e66db52019-04-10 16:24:48 +0000835 // Whatever high bits in c are zero are known to be zero.
Craig Topper8df66c62017-05-12 17:20:30 +0000836 Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros());
Sander de Smalen0e66db52019-04-10 16:24:48 +0000837 }
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000838 break;
839 case ICmpInst::ICMP_ULT:
Sander de Smalen0e66db52019-04-10 16:24:48 +0000840 // assume(v <_u c)
Sander de Smalen0e66db52019-04-10 16:24:48 +0000841 if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
842 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
843 KnownBits RHSKnown(BitWidth);
844 computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I));
845
846 // If the RHS is known zero, then this assumption must be wrong (nothing
847 // is unsigned less than zero). Signal a conflict and get out of here.
848 if (RHSKnown.isZero()) {
849 Known.Zero.setAllBits();
850 Known.One.setAllBits();
851 break;
852 }
853
854 // Whatever high bits in c are zero are known to be zero (if c is a power
855 // of 2, then one more).
856 if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, I)))
857 Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros() + 1);
858 else
859 Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros());
860 }
Sander de Smalen4f5d2df2019-04-11 13:02:19 +0000861 break;
Hal Finkel60db0582014-09-07 18:57:58 +0000862 }
863 }
Sanjay Patel25f6d712017-02-01 15:41:32 +0000864
865 // If assumptions conflict with each other or previous known bits, then we
Sanjay Patel54656ca2017-02-06 18:26:06 +0000866 // have a logical fallacy. It's possible that the assumption is not reachable,
867 // so this isn't a real bug. On the other hand, the program may have undefined
868 // behavior, or we might have a bug in the compiler. We can't assert/crash, so
869 // clear out the known bits, try to warn the user, and hope for the best.
Craig Topperb45eabc2017-04-26 16:39:58 +0000870 if (Known.Zero.intersects(Known.One)) {
Craig Topperf0aeee02017-05-05 17:36:09 +0000871 Known.resetAll();
Sanjay Patel54656ca2017-02-06 18:26:06 +0000872
Vivek Pandya95906582017-10-11 17:12:59 +0000873 if (Q.ORE)
874 Q.ORE->emit([&]() {
875 auto *CxtI = const_cast<Instruction *>(Q.CxtI);
876 return OptimizationRemarkAnalysis("value-tracking", "BadAssumption",
877 CxtI)
878 << "Detected conflicting code assumptions. Program may "
879 "have undefined behavior, or compiler may have "
880 "internal error.";
881 });
Sanjay Patel25f6d712017-02-01 15:41:32 +0000882 }
Hal Finkel60db0582014-09-07 18:57:58 +0000883}
884
Sanjay Patelb7d12382017-10-16 14:46:37 +0000885/// Compute known bits from a shift operator, including those with a
886/// non-constant shift amount. Known is the output of this function. Known2 is a
887/// pre-allocated temporary with the same bit width as Known. KZF and KOF are
Vedant Kumard3196742018-02-28 19:08:52 +0000888/// operator-specific functions that, given the known-zero or known-one bits
Sanjay Patelb7d12382017-10-16 14:46:37 +0000889/// respectively, and a shift amount, compute the implied known-zero or
890/// known-one bits of the shift operator's result respectively for that shift
891/// amount. The results from calling KZF and KOF are conservatively combined for
892/// all permitted shift amounts.
David Majnemer54690dc2016-08-23 20:52:00 +0000893static void computeKnownBitsFromShiftOperator(
Craig Topperb45eabc2017-04-26 16:39:58 +0000894 const Operator *I, KnownBits &Known, KnownBits &Known2,
895 unsigned Depth, const Query &Q,
Sam McCalld0d43e62017-12-04 12:51:49 +0000896 function_ref<APInt(const APInt &, unsigned)> KZF,
897 function_ref<APInt(const APInt &, unsigned)> KOF) {
Craig Topperb45eabc2017-04-26 16:39:58 +0000898 unsigned BitWidth = Known.getBitWidth();
Hal Finkelf2199b22015-10-23 20:37:08 +0000899
900 if (auto *SA = dyn_cast<ConstantInt>(I->getOperand(1))) {
901 unsigned ShiftAmt = SA->getLimitedValue(BitWidth-1);
902
Craig Topperb45eabc2017-04-26 16:39:58 +0000903 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
Sam McCalld0d43e62017-12-04 12:51:49 +0000904 Known.Zero = KZF(Known.Zero, ShiftAmt);
905 Known.One = KOF(Known.One, ShiftAmt);
Sanjay Patele272be72017-10-12 17:31:46 +0000906 // If the known bits conflict, this must be an overflowing left shift, so
907 // the shift result is poison. We can return anything we want. Choose 0 for
908 // the best folding opportunity.
909 if (Known.hasConflict())
910 Known.setAllZero();
Evgeny Stupachenkod7f9c352016-08-24 23:01:33 +0000911
Hal Finkelf2199b22015-10-23 20:37:08 +0000912 return;
913 }
914
Craig Topperb45eabc2017-04-26 16:39:58 +0000915 computeKnownBits(I->getOperand(1), Known, Depth + 1, Q);
Hal Finkelf2199b22015-10-23 20:37:08 +0000916
Sanjay Patele272be72017-10-12 17:31:46 +0000917 // If the shift amount could be greater than or equal to the bit-width of the
918 // LHS, the value could be poison, but bail out because the check below is
919 // expensive. TODO: Should we just carry on?
Roman Lebedev9a20c792019-12-03 17:45:47 +0300920 if (Known.getMaxValue().uge(BitWidth)) {
Craig Topperf0aeee02017-05-05 17:36:09 +0000921 Known.resetAll();
Oliver Stannard06204112017-03-14 10:13:17 +0000922 return;
923 }
924
Craig Topperb45eabc2017-04-26 16:39:58 +0000925 // Note: We cannot use Known.Zero.getLimitedValue() here, because if
Hal Finkelf2199b22015-10-23 20:37:08 +0000926 // BitWidth > 64 and any upper bits are known, we'll end up returning the
927 // limit value (which implies all bits are known).
Craig Topperb45eabc2017-04-26 16:39:58 +0000928 uint64_t ShiftAmtKZ = Known.Zero.zextOrTrunc(64).getZExtValue();
929 uint64_t ShiftAmtKO = Known.One.zextOrTrunc(64).getZExtValue();
Hal Finkelf2199b22015-10-23 20:37:08 +0000930
931 // It would be more-clearly correct to use the two temporaries for this
932 // calculation. Reusing the APInts here to prevent unnecessary allocations.
Craig Topperf0aeee02017-05-05 17:36:09 +0000933 Known.resetAll();
Hal Finkelf2199b22015-10-23 20:37:08 +0000934
James Molloy493e57d2015-10-26 14:10:46 +0000935 // If we know the shifter operand is nonzero, we can sometimes infer more
936 // known bits. However this is expensive to compute, so be lazy about it and
937 // only compute it when absolutely necessary.
938 Optional<bool> ShifterOperandIsNonZero;
939
Hal Finkelf2199b22015-10-23 20:37:08 +0000940 // Early exit if we can't constrain any well-defined shift amount.
Craig Topperf93b7b12017-06-14 17:04:59 +0000941 if (!(ShiftAmtKZ & (PowerOf2Ceil(BitWidth) - 1)) &&
942 !(ShiftAmtKO & (PowerOf2Ceil(BitWidth) - 1))) {
Sanjay Patelb7d12382017-10-16 14:46:37 +0000943 ShifterOperandIsNonZero = isKnownNonZero(I->getOperand(1), Depth + 1, Q);
James Molloy493e57d2015-10-26 14:10:46 +0000944 if (!*ShifterOperandIsNonZero)
945 return;
946 }
Hal Finkelf2199b22015-10-23 20:37:08 +0000947
Craig Topperb45eabc2017-04-26 16:39:58 +0000948 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Hal Finkelf2199b22015-10-23 20:37:08 +0000949
Craig Topperb45eabc2017-04-26 16:39:58 +0000950 Known.Zero.setAllBits();
951 Known.One.setAllBits();
Hal Finkelf2199b22015-10-23 20:37:08 +0000952 for (unsigned ShiftAmt = 0; ShiftAmt < BitWidth; ++ShiftAmt) {
953 // Combine the shifted known input bits only for those shift amounts
954 // compatible with its known constraints.
955 if ((ShiftAmt & ~ShiftAmtKZ) != ShiftAmt)
956 continue;
957 if ((ShiftAmt | ShiftAmtKO) != ShiftAmt)
958 continue;
James Molloy493e57d2015-10-26 14:10:46 +0000959 // If we know the shifter is nonzero, we may be able to infer more known
960 // bits. This check is sunk down as far as possible to avoid the expensive
961 // call to isKnownNonZero if the cheaper checks above fail.
962 if (ShiftAmt == 0) {
963 if (!ShifterOperandIsNonZero.hasValue())
964 ShifterOperandIsNonZero =
Matthias Braunfeb81bc2016-01-15 22:22:04 +0000965 isKnownNonZero(I->getOperand(1), Depth + 1, Q);
James Molloy493e57d2015-10-26 14:10:46 +0000966 if (*ShifterOperandIsNonZero)
967 continue;
968 }
Hal Finkelf2199b22015-10-23 20:37:08 +0000969
Sam McCalld0d43e62017-12-04 12:51:49 +0000970 Known.Zero &= KZF(Known2.Zero, ShiftAmt);
971 Known.One &= KOF(Known2.One, ShiftAmt);
Hal Finkelf2199b22015-10-23 20:37:08 +0000972 }
973
Sanjay Patele272be72017-10-12 17:31:46 +0000974 // If the known bits conflict, the result is poison. Return a 0 and hope the
975 // caller can further optimize that.
976 if (Known.hasConflict())
977 Known.setAllZero();
Hal Finkelf2199b22015-10-23 20:37:08 +0000978}
979
Craig Topperb45eabc2017-04-26 16:39:58 +0000980static void computeKnownBitsFromOperator(const Operator *I, KnownBits &Known,
981 unsigned Depth, const Query &Q) {
982 unsigned BitWidth = Known.getBitWidth();
Rafael Espindolaba0a6ca2012-04-04 12:51:34 +0000983
Craig Topperb45eabc2017-04-26 16:39:58 +0000984 KnownBits Known2(Known);
Dan Gohman80ca01c2009-07-17 20:47:02 +0000985 switch (I->getOpcode()) {
Chris Lattner965c7692008-06-02 01:18:21 +0000986 default: break;
Rafael Espindola53190532012-03-30 15:52:11 +0000987 case Instruction::Load:
Florian Hahn19f9e322018-08-17 14:39:04 +0000988 if (MDNode *MD =
989 Q.IIQ.getMetadata(cast<LoadInst>(I), LLVMContext::MD_range))
Craig Topperf42b23f2017-04-28 06:28:56 +0000990 computeKnownBitsFromRangeMetadata(*MD, Known);
Jay Foad5a29c362014-05-15 12:12:55 +0000991 break;
Chris Lattner965c7692008-06-02 01:18:21 +0000992 case Instruction::And: {
993 // If either the LHS or the RHS are Zero, the result is zero.
Craig Topperb45eabc2017-04-26 16:39:58 +0000994 computeKnownBits(I->getOperand(1), Known, Depth + 1, Q);
995 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Craig Topper1bef2c82012-12-22 19:15:35 +0000996
Chris Lattner965c7692008-06-02 01:18:21 +0000997 // Output known-1 bits are only known if set in both the LHS & RHS.
Craig Topperb45eabc2017-04-26 16:39:58 +0000998 Known.One &= Known2.One;
Chris Lattner965c7692008-06-02 01:18:21 +0000999 // Output known-0 are known to be clear if zero in either the LHS | RHS.
Craig Topperb45eabc2017-04-26 16:39:58 +00001000 Known.Zero |= Known2.Zero;
Philip Reames2d858742015-11-10 18:46:14 +00001001
1002 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1003 // here we handle the more general case of adding any odd number by
1004 // matching the form add(x, add(x, y)) where y is odd.
1005 // TODO: This could be generalized to clearing any bit set in y where the
1006 // following bit is known to be unset in y.
Roman Lebedev6959b8e2018-04-27 21:23:20 +00001007 Value *X = nullptr, *Y = nullptr;
Craig Topperb45eabc2017-04-26 16:39:58 +00001008 if (!Known.Zero[0] && !Known.One[0] &&
Roman Lebedev6959b8e2018-04-27 21:23:20 +00001009 match(I, m_c_BinOp(m_Value(X), m_Add(m_Deferred(X), m_Value(Y))))) {
Craig Topperf0aeee02017-05-05 17:36:09 +00001010 Known2.resetAll();
Craig Topperb45eabc2017-04-26 16:39:58 +00001011 computeKnownBits(Y, Known2, Depth + 1, Q);
Craig Topper8df66c62017-05-12 17:20:30 +00001012 if (Known2.countMinTrailingOnes() > 0)
Craig Topperb45eabc2017-04-26 16:39:58 +00001013 Known.Zero.setBit(0);
Philip Reames2d858742015-11-10 18:46:14 +00001014 }
Jay Foad5a29c362014-05-15 12:12:55 +00001015 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001016 }
Eugene Zelenko75075ef2017-09-01 21:37:29 +00001017 case Instruction::Or:
Craig Topperb45eabc2017-04-26 16:39:58 +00001018 computeKnownBits(I->getOperand(1), Known, Depth + 1, Q);
1019 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Craig Topper1bef2c82012-12-22 19:15:35 +00001020
Chris Lattner965c7692008-06-02 01:18:21 +00001021 // Output known-0 bits are only known if clear in both the LHS & RHS.
Craig Topperb45eabc2017-04-26 16:39:58 +00001022 Known.Zero &= Known2.Zero;
Chris Lattner965c7692008-06-02 01:18:21 +00001023 // Output known-1 are known to be set if set in either the LHS | RHS.
Craig Topperb45eabc2017-04-26 16:39:58 +00001024 Known.One |= Known2.One;
Jay Foad5a29c362014-05-15 12:12:55 +00001025 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001026 case Instruction::Xor: {
Craig Topperb45eabc2017-04-26 16:39:58 +00001027 computeKnownBits(I->getOperand(1), Known, Depth + 1, Q);
1028 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Craig Topper1bef2c82012-12-22 19:15:35 +00001029
Chris Lattner965c7692008-06-02 01:18:21 +00001030 // Output known-0 bits are known if clear or set in both the LHS & RHS.
Craig Topperb45eabc2017-04-26 16:39:58 +00001031 APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
Chris Lattner965c7692008-06-02 01:18:21 +00001032 // Output known-1 are known to be set if set in only one of the LHS, RHS.
Craig Topperb45eabc2017-04-26 16:39:58 +00001033 Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
1034 Known.Zero = std::move(KnownZeroOut);
Jay Foad5a29c362014-05-15 12:12:55 +00001035 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001036 }
1037 case Instruction::Mul: {
Florian Hahn19f9e322018-08-17 14:39:04 +00001038 bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
Craig Topperb45eabc2017-04-26 16:39:58 +00001039 computeKnownBitsMul(I->getOperand(0), I->getOperand(1), NSW, Known,
1040 Known2, Depth, Q);
Nick Lewyckyfa306072012-03-18 23:28:48 +00001041 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001042 }
1043 case Instruction::UDiv: {
1044 // For the purposes of computing leading zeros we can conservatively
1045 // treat a udiv as a logical right shift by the power of 2 known to
1046 // be less than the denominator.
Craig Topperb45eabc2017-04-26 16:39:58 +00001047 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Craig Topper8df66c62017-05-12 17:20:30 +00001048 unsigned LeadZ = Known2.countMinLeadingZeros();
Chris Lattner965c7692008-06-02 01:18:21 +00001049
Craig Topperf0aeee02017-05-05 17:36:09 +00001050 Known2.resetAll();
Craig Topperb45eabc2017-04-26 16:39:58 +00001051 computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q);
Craig Topper8df66c62017-05-12 17:20:30 +00001052 unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
1053 if (RHSMaxLeadingZeros != BitWidth)
1054 LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
Chris Lattner965c7692008-06-02 01:18:21 +00001055
Craig Topperb45eabc2017-04-26 16:39:58 +00001056 Known.Zero.setHighBits(LeadZ);
Jay Foad5a29c362014-05-15 12:12:55 +00001057 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001058 }
David Majnemera19d0f22016-08-06 08:16:00 +00001059 case Instruction::Select: {
Simon Pilgrimf62293e2019-09-23 13:15:52 +00001060 const Value *LHS = nullptr, *RHS = nullptr;
David Majnemera19d0f22016-08-06 08:16:00 +00001061 SelectPatternFlavor SPF = matchSelectPattern(I, LHS, RHS).Flavor;
1062 if (SelectPatternResult::isMinOrMax(SPF)) {
Craig Topperb45eabc2017-04-26 16:39:58 +00001063 computeKnownBits(RHS, Known, Depth + 1, Q);
1064 computeKnownBits(LHS, Known2, Depth + 1, Q);
David Majnemera19d0f22016-08-06 08:16:00 +00001065 } else {
Craig Topperb45eabc2017-04-26 16:39:58 +00001066 computeKnownBits(I->getOperand(2), Known, Depth + 1, Q);
1067 computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q);
David Majnemera19d0f22016-08-06 08:16:00 +00001068 }
1069
1070 unsigned MaxHighOnes = 0;
1071 unsigned MaxHighZeros = 0;
1072 if (SPF == SPF_SMAX) {
1073 // If both sides are negative, the result is negative.
Craig Topperca48af32017-04-29 16:43:11 +00001074 if (Known.isNegative() && Known2.isNegative())
David Majnemera19d0f22016-08-06 08:16:00 +00001075 // We can derive a lower bound on the result by taking the max of the
1076 // leading one bits.
Craig Topper8df66c62017-05-12 17:20:30 +00001077 MaxHighOnes =
1078 std::max(Known.countMinLeadingOnes(), Known2.countMinLeadingOnes());
David Majnemera19d0f22016-08-06 08:16:00 +00001079 // If either side is non-negative, the result is non-negative.
Craig Topperca48af32017-04-29 16:43:11 +00001080 else if (Known.isNonNegative() || Known2.isNonNegative())
David Majnemera19d0f22016-08-06 08:16:00 +00001081 MaxHighZeros = 1;
1082 } else if (SPF == SPF_SMIN) {
1083 // If both sides are non-negative, the result is non-negative.
Craig Topperca48af32017-04-29 16:43:11 +00001084 if (Known.isNonNegative() && Known2.isNonNegative())
David Majnemera19d0f22016-08-06 08:16:00 +00001085 // We can derive an upper bound on the result by taking the max of the
1086 // leading zero bits.
Craig Topper8df66c62017-05-12 17:20:30 +00001087 MaxHighZeros = std::max(Known.countMinLeadingZeros(),
1088 Known2.countMinLeadingZeros());
David Majnemera19d0f22016-08-06 08:16:00 +00001089 // If either side is negative, the result is negative.
Craig Topperca48af32017-04-29 16:43:11 +00001090 else if (Known.isNegative() || Known2.isNegative())
David Majnemera19d0f22016-08-06 08:16:00 +00001091 MaxHighOnes = 1;
1092 } else if (SPF == SPF_UMAX) {
1093 // We can derive a lower bound on the result by taking the max of the
1094 // leading one bits.
1095 MaxHighOnes =
Craig Topper8df66c62017-05-12 17:20:30 +00001096 std::max(Known.countMinLeadingOnes(), Known2.countMinLeadingOnes());
David Majnemera19d0f22016-08-06 08:16:00 +00001097 } else if (SPF == SPF_UMIN) {
1098 // We can derive an upper bound on the result by taking the max of the
1099 // leading zero bits.
1100 MaxHighZeros =
Craig Topper8df66c62017-05-12 17:20:30 +00001101 std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
Craig Topper8f77dca2018-05-25 19:18:09 +00001102 } else if (SPF == SPF_ABS) {
1103 // RHS from matchSelectPattern returns the negation part of abs pattern.
1104 // If the negate has an NSW flag we can assume the sign bit of the result
1105 // will be 0 because that makes abs(INT_MIN) undefined.
Craig Topper66c08432019-08-07 18:28:16 +00001106 if (match(RHS, m_Neg(m_Specific(LHS))) &&
1107 Q.IIQ.hasNoSignedWrap(cast<Instruction>(RHS)))
Craig Topper8f77dca2018-05-25 19:18:09 +00001108 MaxHighZeros = 1;
David Majnemera19d0f22016-08-06 08:16:00 +00001109 }
1110
Chris Lattner965c7692008-06-02 01:18:21 +00001111 // Only known if known in both the LHS and RHS.
Craig Topperb45eabc2017-04-26 16:39:58 +00001112 Known.One &= Known2.One;
1113 Known.Zero &= Known2.Zero;
David Majnemera19d0f22016-08-06 08:16:00 +00001114 if (MaxHighOnes > 0)
Craig Topperb45eabc2017-04-26 16:39:58 +00001115 Known.One.setHighBits(MaxHighOnes);
David Majnemera19d0f22016-08-06 08:16:00 +00001116 if (MaxHighZeros > 0)
Craig Topperb45eabc2017-04-26 16:39:58 +00001117 Known.Zero.setHighBits(MaxHighZeros);
Jay Foad5a29c362014-05-15 12:12:55 +00001118 break;
David Majnemera19d0f22016-08-06 08:16:00 +00001119 }
Chris Lattner965c7692008-06-02 01:18:21 +00001120 case Instruction::FPTrunc:
1121 case Instruction::FPExt:
1122 case Instruction::FPToUI:
1123 case Instruction::FPToSI:
1124 case Instruction::SIToFP:
1125 case Instruction::UIToFP:
Jay Foad5a29c362014-05-15 12:12:55 +00001126 break; // Can't work with floating point.
Chris Lattner965c7692008-06-02 01:18:21 +00001127 case Instruction::PtrToInt:
1128 case Instruction::IntToPtr:
Justin Bognercd1d5aa2016-08-17 20:30:52 +00001129 // Fall through and handle them the same as zext/trunc.
1130 LLVM_FALLTHROUGH;
Chris Lattner965c7692008-06-02 01:18:21 +00001131 case Instruction::ZExt:
1132 case Instruction::Trunc: {
Chris Lattner229907c2011-07-18 04:54:35 +00001133 Type *SrcTy = I->getOperand(0)->getType();
Nadav Rotem15198e92012-10-26 17:17:05 +00001134
Chris Lattner0cdbc7a2009-09-08 00:13:52 +00001135 unsigned SrcBitWidth;
Chris Lattner965c7692008-06-02 01:18:21 +00001136 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1137 // which fall through here.
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00001138 Type *ScalarTy = SrcTy->getScalarType();
1139 SrcBitWidth = ScalarTy->isPointerTy() ?
Nicola Zaghenf798eb22019-12-12 10:25:14 +00001140 Q.DL.getIndexTypeSizeInBits(ScalarTy) :
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00001141 Q.DL.getTypeSizeInBits(ScalarTy);
Nadav Rotem15198e92012-10-26 17:17:05 +00001142
1143 assert(SrcBitWidth && "SrcBitWidth can't be zero");
Bjorn Petterssond30f3082019-02-28 15:45:29 +00001144 Known = Known.zextOrTrunc(SrcBitWidth, false);
Craig Topperb45eabc2017-04-26 16:39:58 +00001145 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
Bjorn Petterssond30f3082019-02-28 15:45:29 +00001146 Known = Known.zextOrTrunc(BitWidth, true /* ExtendedBitsAreKnownZero */);
Jay Foad5a29c362014-05-15 12:12:55 +00001147 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001148 }
1149 case Instruction::BitCast: {
Chris Lattner229907c2011-07-18 04:54:35 +00001150 Type *SrcTy = I->getOperand(0)->getType();
Vedant Kumarb3091da2018-07-06 20:17:42 +00001151 if (SrcTy->isIntOrPtrTy() &&
Chris Lattneredb84072009-07-02 16:04:08 +00001152 // TODO: For now, not handling conversions like:
1153 // (bitcast i64 %x to <2 x i32>)
Duncan Sands19d0b472010-02-16 11:11:14 +00001154 !I->getType()->isVectorTy()) {
Craig Topperb45eabc2017-04-26 16:39:58 +00001155 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
Jay Foad5a29c362014-05-15 12:12:55 +00001156 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001157 }
1158 break;
1159 }
1160 case Instruction::SExt: {
1161 // Compute the bits in the result that are not present in the input.
Chris Lattner0cdbc7a2009-09-08 00:13:52 +00001162 unsigned SrcBitWidth = I->getOperand(0)->getType()->getScalarSizeInBits();
Craig Topper1bef2c82012-12-22 19:15:35 +00001163
Craig Topperd938fd12017-05-03 22:07:25 +00001164 Known = Known.trunc(SrcBitWidth);
Craig Topperb45eabc2017-04-26 16:39:58 +00001165 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
Chris Lattner965c7692008-06-02 01:18:21 +00001166 // If the sign bit of the input is known set or clear, then we know the
1167 // top bits of the result.
Craig Topperd938fd12017-05-03 22:07:25 +00001168 Known = Known.sext(BitWidth);
Jay Foad5a29c362014-05-15 12:12:55 +00001169 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001170 }
Hal Finkelf2199b22015-10-23 20:37:08 +00001171 case Instruction::Shl: {
Sylvestre Ledru91ce36c2012-09-27 10:14:43 +00001172 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
Florian Hahn19f9e322018-08-17 14:39:04 +00001173 bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
Sam McCalld0d43e62017-12-04 12:51:49 +00001174 auto KZF = [NSW](const APInt &KnownZero, unsigned ShiftAmt) {
1175 APInt KZResult = KnownZero << ShiftAmt;
1176 KZResult.setLowBits(ShiftAmt); // Low bits known 0.
Evgeny Stupachenkod7f9c352016-08-24 23:01:33 +00001177 // If this shift has "nsw" keyword, then the result is either a poison
1178 // value or has the same sign bit as the first operand.
Sam McCalld0d43e62017-12-04 12:51:49 +00001179 if (NSW && KnownZero.isSignBitSet())
1180 KZResult.setSignBit();
1181 return KZResult;
Hal Finkelf2199b22015-10-23 20:37:08 +00001182 };
1183
Sam McCalld0d43e62017-12-04 12:51:49 +00001184 auto KOF = [NSW](const APInt &KnownOne, unsigned ShiftAmt) {
1185 APInt KOResult = KnownOne << ShiftAmt;
1186 if (NSW && KnownOne.isSignBitSet())
1187 KOResult.setSignBit();
1188 return KOResult;
1189 };
1190
1191 computeKnownBitsFromShiftOperator(I, Known, Known2, Depth, Q, KZF, KOF);
Chris Lattner965c7692008-06-02 01:18:21 +00001192 break;
Hal Finkelf2199b22015-10-23 20:37:08 +00001193 }
1194 case Instruction::LShr: {
Sanjay Patelb7d12382017-10-16 14:46:37 +00001195 // (lshr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
Sam McCalld0d43e62017-12-04 12:51:49 +00001196 auto KZF = [](const APInt &KnownZero, unsigned ShiftAmt) {
1197 APInt KZResult = KnownZero.lshr(ShiftAmt);
1198 // High bits known zero.
1199 KZResult.setHighBits(ShiftAmt);
1200 return KZResult;
Hal Finkelf2199b22015-10-23 20:37:08 +00001201 };
Craig Topper1bef2c82012-12-22 19:15:35 +00001202
Sam McCalld0d43e62017-12-04 12:51:49 +00001203 auto KOF = [](const APInt &KnownOne, unsigned ShiftAmt) {
1204 return KnownOne.lshr(ShiftAmt);
1205 };
1206
1207 computeKnownBitsFromShiftOperator(I, Known, Known2, Depth, Q, KZF, KOF);
Chris Lattner965c7692008-06-02 01:18:21 +00001208 break;
Hal Finkelf2199b22015-10-23 20:37:08 +00001209 }
1210 case Instruction::AShr: {
Sylvestre Ledru91ce36c2012-09-27 10:14:43 +00001211 // (ashr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
Sam McCalld0d43e62017-12-04 12:51:49 +00001212 auto KZF = [](const APInt &KnownZero, unsigned ShiftAmt) {
1213 return KnownZero.ashr(ShiftAmt);
Hal Finkelf2199b22015-10-23 20:37:08 +00001214 };
Craig Topper1bef2c82012-12-22 19:15:35 +00001215
Sam McCalld0d43e62017-12-04 12:51:49 +00001216 auto KOF = [](const APInt &KnownOne, unsigned ShiftAmt) {
1217 return KnownOne.ashr(ShiftAmt);
1218 };
1219
1220 computeKnownBitsFromShiftOperator(I, Known, Known2, Depth, Q, KZF, KOF);
Chris Lattner965c7692008-06-02 01:18:21 +00001221 break;
Hal Finkelf2199b22015-10-23 20:37:08 +00001222 }
Chris Lattner965c7692008-06-02 01:18:21 +00001223 case Instruction::Sub: {
Florian Hahn19f9e322018-08-17 14:39:04 +00001224 bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
Jay Foada0653a32014-05-14 21:14:37 +00001225 computeKnownBitsAddSub(false, I->getOperand(0), I->getOperand(1), NSW,
Craig Topperb45eabc2017-04-26 16:39:58 +00001226 Known, Known2, Depth, Q);
Nick Lewyckyfea3e002012-03-09 09:23:50 +00001227 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001228 }
Chris Lattner965c7692008-06-02 01:18:21 +00001229 case Instruction::Add: {
Florian Hahn19f9e322018-08-17 14:39:04 +00001230 bool NSW = Q.IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(I));
Jay Foada0653a32014-05-14 21:14:37 +00001231 computeKnownBitsAddSub(true, I->getOperand(0), I->getOperand(1), NSW,
Craig Topperb45eabc2017-04-26 16:39:58 +00001232 Known, Known2, Depth, Q);
Nick Lewyckyfea3e002012-03-09 09:23:50 +00001233 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001234 }
1235 case Instruction::SRem:
1236 if (ConstantInt *Rem = dyn_cast<ConstantInt>(I->getOperand(1))) {
Duncan Sands26cd6bd2010-01-29 06:18:37 +00001237 APInt RA = Rem->getValue().abs();
1238 if (RA.isPowerOf2()) {
1239 APInt LowBits = RA - 1;
Craig Topperb45eabc2017-04-26 16:39:58 +00001240 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Chris Lattner965c7692008-06-02 01:18:21 +00001241
Duncan Sands26cd6bd2010-01-29 06:18:37 +00001242 // The low bits of the first operand are unchanged by the srem.
Craig Topperb45eabc2017-04-26 16:39:58 +00001243 Known.Zero = Known2.Zero & LowBits;
1244 Known.One = Known2.One & LowBits;
Chris Lattner965c7692008-06-02 01:18:21 +00001245
Duncan Sands26cd6bd2010-01-29 06:18:37 +00001246 // If the first operand is non-negative or has all low bits zero, then
1247 // the upper bits are all zero.
Craig Topperca48af32017-04-29 16:43:11 +00001248 if (Known2.isNonNegative() || LowBits.isSubsetOf(Known2.Zero))
Craig Topperb45eabc2017-04-26 16:39:58 +00001249 Known.Zero |= ~LowBits;
Duncan Sands26cd6bd2010-01-29 06:18:37 +00001250
1251 // If the first operand is negative and not all low bits are zero, then
1252 // the upper bits are all one.
Craig Topperca48af32017-04-29 16:43:11 +00001253 if (Known2.isNegative() && LowBits.intersects(Known2.One))
Craig Topperb45eabc2017-04-26 16:39:58 +00001254 Known.One |= ~LowBits;
Duncan Sands26cd6bd2010-01-29 06:18:37 +00001255
Craig Topperb45eabc2017-04-26 16:39:58 +00001256 assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?");
Craig Topperda886c62017-04-16 21:46:12 +00001257 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001258 }
1259 }
Nick Lewyckye4679792011-03-07 01:50:10 +00001260
1261 // The sign bit is the LHS's sign bit, except when the result of the
1262 // remainder is zero.
Craig Topperb45eabc2017-04-26 16:39:58 +00001263 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Craig Topperda886c62017-04-16 21:46:12 +00001264 // If it's known zero, our sign bit is also zero.
Craig Topperca48af32017-04-29 16:43:11 +00001265 if (Known2.isNonNegative())
1266 Known.makeNonNegative();
Nick Lewyckye4679792011-03-07 01:50:10 +00001267
Chris Lattner965c7692008-06-02 01:18:21 +00001268 break;
1269 case Instruction::URem: {
1270 if (ConstantInt *Rem = dyn_cast<ConstantInt>(I->getOperand(1))) {
Benjamin Kramer46e38f32016-06-08 10:01:20 +00001271 const APInt &RA = Rem->getValue();
Chris Lattner965c7692008-06-02 01:18:21 +00001272 if (RA.isPowerOf2()) {
1273 APInt LowBits = (RA - 1);
Craig Topperb45eabc2017-04-26 16:39:58 +00001274 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
1275 Known.Zero |= ~LowBits;
1276 Known.One &= LowBits;
Chris Lattner965c7692008-06-02 01:18:21 +00001277 break;
1278 }
1279 }
1280
1281 // Since the result is less than or equal to either operand, any leading
1282 // zero bits in either operand must also exist in the result.
Craig Topperb45eabc2017-04-26 16:39:58 +00001283 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
1284 computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q);
Chris Lattner965c7692008-06-02 01:18:21 +00001285
Craig Topper8df66c62017-05-12 17:20:30 +00001286 unsigned Leaders =
1287 std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
Craig Topperf0aeee02017-05-05 17:36:09 +00001288 Known.resetAll();
Craig Topperb45eabc2017-04-26 16:39:58 +00001289 Known.Zero.setHighBits(Leaders);
Chris Lattner965c7692008-06-02 01:18:21 +00001290 break;
1291 }
1292
Victor Hernandeza3aaf852009-10-17 01:18:07 +00001293 case Instruction::Alloca: {
Pete Cooper35b00d52016-08-13 01:05:32 +00001294 const AllocaInst *AI = cast<AllocaInst>(I);
Chris Lattner965c7692008-06-02 01:18:21 +00001295 unsigned Align = AI->getAlignment();
Mehdi Aminia28d91d2015-03-10 02:37:25 +00001296 if (Align == 0)
Eduard Burtescu90c44492016-01-18 00:10:01 +00001297 Align = Q.DL.getABITypeAlignment(AI->getAllocatedType());
Craig Topper1bef2c82012-12-22 19:15:35 +00001298
Chris Lattner965c7692008-06-02 01:18:21 +00001299 if (Align > 0)
Craig Topperb45eabc2017-04-26 16:39:58 +00001300 Known.Zero.setLowBits(countTrailingZeros(Align));
Chris Lattner965c7692008-06-02 01:18:21 +00001301 break;
1302 }
1303 case Instruction::GetElementPtr: {
1304 // Analyze all of the subscripts of this getelementptr instruction
1305 // to determine if we can prove known low zero bits.
Craig Topperb45eabc2017-04-26 16:39:58 +00001306 KnownBits LocalKnown(BitWidth);
1307 computeKnownBits(I->getOperand(0), LocalKnown, Depth + 1, Q);
Craig Topper8df66c62017-05-12 17:20:30 +00001308 unsigned TrailZ = LocalKnown.countMinTrailingZeros();
Chris Lattner965c7692008-06-02 01:18:21 +00001309
1310 gep_type_iterator GTI = gep_type_begin(I);
1311 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1312 Value *Index = I->getOperand(i);
Peter Collingbourneab85225b2016-12-02 02:24:42 +00001313 if (StructType *STy = GTI.getStructTypeOrNull()) {
Chris Lattner965c7692008-06-02 01:18:21 +00001314 // Handle struct member offset arithmetic.
Matt Arsenault74742a12013-08-19 21:43:16 +00001315
1316 // Handle case when index is vector zeroinitializer
1317 Constant *CIndex = cast<Constant>(Index);
1318 if (CIndex->isZeroValue())
1319 continue;
1320
1321 if (CIndex->getType()->isVectorTy())
1322 Index = CIndex->getSplatValue();
1323
Chris Lattner965c7692008-06-02 01:18:21 +00001324 unsigned Idx = cast<ConstantInt>(Index)->getZExtValue();
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001325 const StructLayout *SL = Q.DL.getStructLayout(STy);
Chris Lattner965c7692008-06-02 01:18:21 +00001326 uint64_t Offset = SL->getElementOffset(Idx);
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00001327 TrailZ = std::min<unsigned>(TrailZ,
1328 countTrailingZeros(Offset));
Chris Lattner965c7692008-06-02 01:18:21 +00001329 } else {
1330 // Handle array index arithmetic.
Chris Lattner229907c2011-07-18 04:54:35 +00001331 Type *IndexedTy = GTI.getIndexedType();
Jay Foad5a29c362014-05-15 12:12:55 +00001332 if (!IndexedTy->isSized()) {
1333 TrailZ = 0;
1334 break;
1335 }
Dan Gohman7ccc52f2009-06-15 22:12:54 +00001336 unsigned GEPOpiBits = Index->getType()->getScalarSizeInBits();
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001337 uint64_t TypeSize = Q.DL.getTypeAllocSize(IndexedTy);
Craig Topperb45eabc2017-04-26 16:39:58 +00001338 LocalKnown.Zero = LocalKnown.One = APInt(GEPOpiBits, 0);
1339 computeKnownBits(Index, LocalKnown, Depth + 1, Q);
Chris Lattner965c7692008-06-02 01:18:21 +00001340 TrailZ = std::min(TrailZ,
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00001341 unsigned(countTrailingZeros(TypeSize) +
Craig Topper8df66c62017-05-12 17:20:30 +00001342 LocalKnown.countMinTrailingZeros()));
Chris Lattner965c7692008-06-02 01:18:21 +00001343 }
1344 }
Craig Topper1bef2c82012-12-22 19:15:35 +00001345
Craig Topperb45eabc2017-04-26 16:39:58 +00001346 Known.Zero.setLowBits(TrailZ);
Chris Lattner965c7692008-06-02 01:18:21 +00001347 break;
1348 }
1349 case Instruction::PHI: {
Pete Cooper35b00d52016-08-13 01:05:32 +00001350 const PHINode *P = cast<PHINode>(I);
Chris Lattner965c7692008-06-02 01:18:21 +00001351 // Handle the case of a simple two-predecessor recurrence PHI.
1352 // There's a lot more that could theoretically be done here, but
1353 // this is sufficient to catch some interesting cases.
1354 if (P->getNumIncomingValues() == 2) {
1355 for (unsigned i = 0; i != 2; ++i) {
1356 Value *L = P->getIncomingValue(i);
1357 Value *R = P->getIncomingValue(!i);
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001358 Instruction *RInst = P->getIncomingBlock(!i)->getTerminator();
1359 Instruction *LInst = P->getIncomingBlock(i)->getTerminator();
Dan Gohman80ca01c2009-07-17 20:47:02 +00001360 Operator *LU = dyn_cast<Operator>(L);
Chris Lattner965c7692008-06-02 01:18:21 +00001361 if (!LU)
1362 continue;
Dan Gohman80ca01c2009-07-17 20:47:02 +00001363 unsigned Opcode = LU->getOpcode();
Chris Lattner965c7692008-06-02 01:18:21 +00001364 // Check for operations that have the property that if
1365 // both their operands have low zero bits, the result
Artur Pilipenkobc76eca2016-08-22 13:14:07 +00001366 // will have low zero bits.
Chris Lattner965c7692008-06-02 01:18:21 +00001367 if (Opcode == Instruction::Add ||
1368 Opcode == Instruction::Sub ||
1369 Opcode == Instruction::And ||
1370 Opcode == Instruction::Or ||
1371 Opcode == Instruction::Mul) {
1372 Value *LL = LU->getOperand(0);
1373 Value *LR = LU->getOperand(1);
1374 // Find a recurrence.
1375 if (LL == I)
1376 L = LR;
1377 else if (LR == I)
1378 L = LL;
1379 else
Florian Hahn75be1a92019-08-16 09:15:02 +00001380 continue; // Check for recurrence with L and R flipped.
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001381
1382 // Change the context instruction to the "edge" that flows into the
1383 // phi. This is important because that is where the value is actually
1384 // "evaluated" even though it is used later somewhere else. (see also
1385 // D69571).
1386 Query RecQ = Q;
1387
Chris Lattner965c7692008-06-02 01:18:21 +00001388 // Ok, we have a PHI of the form L op= R. Check for low
1389 // zero bits.
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001390 RecQ.CxtI = RInst;
1391 computeKnownBits(R, Known2, Depth + 1, RecQ);
David Greeneaebd9e02008-10-27 23:24:03 +00001392
1393 // We need to take the minimum number of known bits
Craig Topperb45eabc2017-04-26 16:39:58 +00001394 KnownBits Known3(Known);
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001395 RecQ.CxtI = LInst;
1396 computeKnownBits(L, Known3, Depth + 1, RecQ);
David Greeneaebd9e02008-10-27 23:24:03 +00001397
Craig Topper8df66c62017-05-12 17:20:30 +00001398 Known.Zero.setLowBits(std::min(Known2.countMinTrailingZeros(),
1399 Known3.countMinTrailingZeros()));
Artur Pilipenkoc6eb6bd2016-10-12 16:18:43 +00001400
Artur Pilipenkoc6eb6bd2016-10-12 16:18:43 +00001401 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(LU);
Florian Hahn19f9e322018-08-17 14:39:04 +00001402 if (OverflowOp && Q.IIQ.hasNoSignedWrap(OverflowOp)) {
Artur Pilipenkoc6eb6bd2016-10-12 16:18:43 +00001403 // If initial value of recurrence is nonnegative, and we are adding
1404 // a nonnegative number with nsw, the result can only be nonnegative
1405 // or poison value regardless of the number of times we execute the
1406 // add in phi recurrence. If initial value is negative and we are
1407 // adding a negative number with nsw, the result can only be
1408 // negative or poison value. Similar arguments apply to sub and mul.
1409 //
1410 // (add non-negative, non-negative) --> non-negative
1411 // (add negative, negative) --> negative
1412 if (Opcode == Instruction::Add) {
Craig Topperca48af32017-04-29 16:43:11 +00001413 if (Known2.isNonNegative() && Known3.isNonNegative())
1414 Known.makeNonNegative();
1415 else if (Known2.isNegative() && Known3.isNegative())
1416 Known.makeNegative();
Artur Pilipenkoc6eb6bd2016-10-12 16:18:43 +00001417 }
1418
1419 // (sub nsw non-negative, negative) --> non-negative
1420 // (sub nsw negative, non-negative) --> negative
1421 else if (Opcode == Instruction::Sub && LL == I) {
Craig Topperca48af32017-04-29 16:43:11 +00001422 if (Known2.isNonNegative() && Known3.isNegative())
1423 Known.makeNonNegative();
1424 else if (Known2.isNegative() && Known3.isNonNegative())
1425 Known.makeNegative();
Artur Pilipenkoc6eb6bd2016-10-12 16:18:43 +00001426 }
1427
1428 // (mul nsw non-negative, non-negative) --> non-negative
Craig Topperca48af32017-04-29 16:43:11 +00001429 else if (Opcode == Instruction::Mul && Known2.isNonNegative() &&
1430 Known3.isNonNegative())
1431 Known.makeNonNegative();
Artur Pilipenkoc6eb6bd2016-10-12 16:18:43 +00001432 }
1433
Chris Lattner965c7692008-06-02 01:18:21 +00001434 break;
1435 }
1436 }
1437 }
Dan Gohmanbf0002e2009-05-21 02:28:33 +00001438
Nick Lewyckyac0b62c2011-02-10 23:54:10 +00001439 // Unreachable blocks may have zero-operand PHI nodes.
1440 if (P->getNumIncomingValues() == 0)
Jay Foad5a29c362014-05-15 12:12:55 +00001441 break;
Nick Lewyckyac0b62c2011-02-10 23:54:10 +00001442
Dan Gohmanbf0002e2009-05-21 02:28:33 +00001443 // Otherwise take the unions of the known bit sets of the operands,
1444 // taking conservative care to avoid excessive recursion.
Craig Topperb45eabc2017-04-26 16:39:58 +00001445 if (Depth < MaxDepth - 1 && !Known.Zero && !Known.One) {
Duncan Sands7dc3d472011-03-08 12:39:03 +00001446 // Skip if every incoming value references to ourself.
Nuno Lopes0d44a502012-07-03 21:15:40 +00001447 if (dyn_cast_or_null<UndefValue>(P->hasConstantValue()))
Duncan Sands7dc3d472011-03-08 12:39:03 +00001448 break;
1449
Craig Topperb45eabc2017-04-26 16:39:58 +00001450 Known.Zero.setAllBits();
1451 Known.One.setAllBits();
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001452 for (unsigned u = 0, e = P->getNumIncomingValues(); u < e; ++u) {
1453 Value *IncValue = P->getIncomingValue(u);
Dan Gohmanbf0002e2009-05-21 02:28:33 +00001454 // Skip direct self references.
Pete Cooper833f34d2015-05-12 20:05:31 +00001455 if (IncValue == P) continue;
Dan Gohmanbf0002e2009-05-21 02:28:33 +00001456
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001457 // Change the context instruction to the "edge" that flows into the
1458 // phi. This is important because that is where the value is actually
1459 // "evaluated" even though it is used later somewhere else. (see also
1460 // D69571).
1461 Query RecQ = Q;
1462 RecQ.CxtI = P->getIncomingBlock(u)->getTerminator();
1463
Craig Topperb45eabc2017-04-26 16:39:58 +00001464 Known2 = KnownBits(BitWidth);
Dan Gohmanbf0002e2009-05-21 02:28:33 +00001465 // Recurse, but cap the recursion to one level, because we don't
1466 // want to waste time spinning around in loops.
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001467 computeKnownBits(IncValue, Known2, MaxDepth - 1, RecQ);
Craig Topperb45eabc2017-04-26 16:39:58 +00001468 Known.Zero &= Known2.Zero;
1469 Known.One &= Known2.One;
Dan Gohmanbf0002e2009-05-21 02:28:33 +00001470 // If all bits have been ruled out, there's no need to check
1471 // more operands.
Craig Topperb45eabc2017-04-26 16:39:58 +00001472 if (!Known.Zero && !Known.One)
Dan Gohmanbf0002e2009-05-21 02:28:33 +00001473 break;
1474 }
1475 }
Chris Lattner965c7692008-06-02 01:18:21 +00001476 break;
1477 }
1478 case Instruction::Call:
Jingyue Wu37fcb592014-06-19 16:50:16 +00001479 case Instruction::Invoke:
Hal Finkel6fd5e1f2016-07-11 02:25:14 +00001480 // If range metadata is attached to this call, set known bits from that,
1481 // and then intersect with known bits based on other properties of the
1482 // function.
Florian Hahn19f9e322018-08-17 14:39:04 +00001483 if (MDNode *MD =
1484 Q.IIQ.getMetadata(cast<Instruction>(I), LLVMContext::MD_range))
Craig Topperf42b23f2017-04-28 06:28:56 +00001485 computeKnownBitsFromRangeMetadata(*MD, Known);
Pete Cooper35b00d52016-08-13 01:05:32 +00001486 if (const Value *RV = ImmutableCallSite(I).getReturnedArgOperand()) {
Craig Topperb45eabc2017-04-26 16:39:58 +00001487 computeKnownBits(RV, Known2, Depth + 1, Q);
1488 Known.Zero |= Known2.Zero;
1489 Known.One |= Known2.One;
Hal Finkel6fd5e1f2016-07-11 02:25:14 +00001490 }
Pete Cooper35b00d52016-08-13 01:05:32 +00001491 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
Chris Lattner965c7692008-06-02 01:18:21 +00001492 switch (II->getIntrinsicID()) {
1493 default: break;
Chad Rosier85204292017-01-17 17:23:51 +00001494 case Intrinsic::bitreverse:
Craig Topperb45eabc2017-04-26 16:39:58 +00001495 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
1496 Known.Zero |= Known2.Zero.reverseBits();
1497 Known.One |= Known2.One.reverseBits();
Chad Rosier85204292017-01-17 17:23:51 +00001498 break;
Philip Reames675418e2015-10-06 20:20:45 +00001499 case Intrinsic::bswap:
Craig Topperb45eabc2017-04-26 16:39:58 +00001500 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
1501 Known.Zero |= Known2.Zero.byteSwap();
1502 Known.One |= Known2.One.byteSwap();
Philip Reames675418e2015-10-06 20:20:45 +00001503 break;
Craig Topper868813f2017-05-08 17:22:34 +00001504 case Intrinsic::ctlz: {
1505 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
1506 // If we have a known 1, its position is our upper bound.
1507 unsigned PossibleLZ = Known2.One.countLeadingZeros();
Benjamin Kramer4ee57472011-12-24 17:31:46 +00001508 // If this call is undefined for 0, the result will be less than 2^n.
1509 if (II->getArgOperand(1) == ConstantInt::getTrue(II->getContext()))
Craig Topper868813f2017-05-08 17:22:34 +00001510 PossibleLZ = std::min(PossibleLZ, BitWidth - 1);
1511 unsigned LowBits = Log2_32(PossibleLZ)+1;
1512 Known.Zero.setBitsFrom(LowBits);
1513 break;
1514 }
1515 case Intrinsic::cttz: {
1516 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
1517 // If we have a known 1, its position is our upper bound.
1518 unsigned PossibleTZ = Known2.One.countTrailingZeros();
1519 // If this call is undefined for 0, the result will be less than 2^n.
1520 if (II->getArgOperand(1) == ConstantInt::getTrue(II->getContext()))
1521 PossibleTZ = std::min(PossibleTZ, BitWidth - 1);
1522 unsigned LowBits = Log2_32(PossibleTZ)+1;
Craig Topperb45eabc2017-04-26 16:39:58 +00001523 Known.Zero.setBitsFrom(LowBits);
Benjamin Kramer4ee57472011-12-24 17:31:46 +00001524 break;
1525 }
1526 case Intrinsic::ctpop: {
Craig Topperb45eabc2017-04-26 16:39:58 +00001527 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
Philip Reamesddcf6b32015-10-14 22:42:12 +00001528 // We can bound the space the count needs. Also, bits known to be zero
1529 // can't contribute to the population.
Craig Topper8df66c62017-05-12 17:20:30 +00001530 unsigned BitsPossiblySet = Known2.countMaxPopulation();
Craig Topper66df10f2017-04-14 06:43:34 +00001531 unsigned LowBits = Log2_32(BitsPossiblySet)+1;
Craig Topperb45eabc2017-04-26 16:39:58 +00001532 Known.Zero.setBitsFrom(LowBits);
Philip Reamesddcf6b32015-10-14 22:42:12 +00001533 // TODO: we could bound KnownOne using the lower bound on the number
1534 // of bits which might be set provided by popcnt KnownOne2.
Chris Lattner965c7692008-06-02 01:18:21 +00001535 break;
1536 }
Nikita Popov687b92c2018-12-02 14:14:11 +00001537 case Intrinsic::fshr:
1538 case Intrinsic::fshl: {
1539 const APInt *SA;
1540 if (!match(I->getOperand(2), m_APInt(SA)))
1541 break;
1542
1543 // Normalize to funnel shift left.
1544 uint64_t ShiftAmt = SA->urem(BitWidth);
1545 if (II->getIntrinsicID() == Intrinsic::fshr)
1546 ShiftAmt = BitWidth - ShiftAmt;
1547
1548 KnownBits Known3(Known);
1549 computeKnownBits(I->getOperand(0), Known2, Depth + 1, Q);
1550 computeKnownBits(I->getOperand(1), Known3, Depth + 1, Q);
1551
1552 Known.Zero =
1553 Known2.Zero.shl(ShiftAmt) | Known3.Zero.lshr(BitWidth - ShiftAmt);
1554 Known.One =
1555 Known2.One.shl(ShiftAmt) | Known3.One.lshr(BitWidth - ShiftAmt);
1556 break;
1557 }
Nikita Popoved3ca922019-03-01 20:07:04 +00001558 case Intrinsic::uadd_sat:
1559 case Intrinsic::usub_sat: {
1560 bool IsAdd = II->getIntrinsicID() == Intrinsic::uadd_sat;
1561 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
1562 computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q);
1563
1564 // Add: Leading ones of either operand are preserved.
1565 // Sub: Leading zeros of LHS and leading ones of RHS are preserved
1566 // as leading zeros in the result.
1567 unsigned LeadingKnown;
1568 if (IsAdd)
1569 LeadingKnown = std::max(Known.countMinLeadingOnes(),
1570 Known2.countMinLeadingOnes());
1571 else
1572 LeadingKnown = std::max(Known.countMinLeadingZeros(),
1573 Known2.countMinLeadingOnes());
1574
1575 Known = KnownBits::computeForAddSub(
1576 IsAdd, /* NSW */ false, Known, Known2);
1577
1578 // We select between the operation result and all-ones/zero
1579 // respectively, so we can preserve known ones/zeros.
1580 if (IsAdd) {
1581 Known.One.setHighBits(LeadingKnown);
1582 Known.Zero.clearAllBits();
1583 } else {
1584 Known.Zero.setHighBits(LeadingKnown);
1585 Known.One.clearAllBits();
1586 }
1587 break;
1588 }
Chad Rosierb3628842011-05-26 23:13:19 +00001589 case Intrinsic::x86_sse42_crc32_64_64:
Craig Topperb45eabc2017-04-26 16:39:58 +00001590 Known.Zero.setBitsFrom(32);
Evan Cheng2a746bf2011-05-22 18:25:30 +00001591 break;
Chris Lattner965c7692008-06-02 01:18:21 +00001592 }
1593 }
1594 break;
Bjorn Pettersson39616032016-10-06 09:56:21 +00001595 case Instruction::ExtractElement:
1596 // Look through extract element. At the moment we keep this simple and skip
1597 // tracking the specific element. But at least we might find information
1598 // valid for all elements of the vector (for example if vector is sign
1599 // extended, shifted, etc).
Craig Topperb45eabc2017-04-26 16:39:58 +00001600 computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
Bjorn Pettersson39616032016-10-06 09:56:21 +00001601 break;
Nick Lewyckyfea3e002012-03-09 09:23:50 +00001602 case Instruction::ExtractValue:
1603 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I->getOperand(0))) {
Pete Cooper35b00d52016-08-13 01:05:32 +00001604 const ExtractValueInst *EVI = cast<ExtractValueInst>(I);
Nick Lewyckyfea3e002012-03-09 09:23:50 +00001605 if (EVI->getNumIndices() != 1) break;
1606 if (EVI->getIndices()[0] == 0) {
1607 switch (II->getIntrinsicID()) {
1608 default: break;
1609 case Intrinsic::uadd_with_overflow:
1610 case Intrinsic::sadd_with_overflow:
Jay Foada0653a32014-05-14 21:14:37 +00001611 computeKnownBitsAddSub(true, II->getArgOperand(0),
Craig Topperb45eabc2017-04-26 16:39:58 +00001612 II->getArgOperand(1), false, Known, Known2,
1613 Depth, Q);
Nick Lewyckyfea3e002012-03-09 09:23:50 +00001614 break;
1615 case Intrinsic::usub_with_overflow:
1616 case Intrinsic::ssub_with_overflow:
Jay Foada0653a32014-05-14 21:14:37 +00001617 computeKnownBitsAddSub(false, II->getArgOperand(0),
Craig Topperb45eabc2017-04-26 16:39:58 +00001618 II->getArgOperand(1), false, Known, Known2,
1619 Depth, Q);
Nick Lewyckyfea3e002012-03-09 09:23:50 +00001620 break;
Nick Lewyckyfa306072012-03-18 23:28:48 +00001621 case Intrinsic::umul_with_overflow:
1622 case Intrinsic::smul_with_overflow:
Mehdi Aminia28d91d2015-03-10 02:37:25 +00001623 computeKnownBitsMul(II->getArgOperand(0), II->getArgOperand(1), false,
Craig Topperb45eabc2017-04-26 16:39:58 +00001624 Known, Known2, Depth, Q);
Nick Lewyckyfa306072012-03-18 23:28:48 +00001625 break;
Nick Lewyckyfea3e002012-03-09 09:23:50 +00001626 }
1627 }
1628 }
Chris Lattner965c7692008-06-02 01:18:21 +00001629 }
Jingyue Wu12b0c282015-06-15 05:46:29 +00001630}
1631
1632/// Determine which bits of V are known to be either zero or one and return
Craig Topper6e11a052017-05-08 16:22:48 +00001633/// them.
1634KnownBits computeKnownBits(const Value *V, unsigned Depth, const Query &Q) {
1635 KnownBits Known(getBitWidth(V->getType(), Q.DL));
1636 computeKnownBits(V, Known, Depth, Q);
1637 return Known;
1638}
1639
1640/// Determine which bits of V are known to be either zero or one and return
Craig Topperb45eabc2017-04-26 16:39:58 +00001641/// them in the Known bit set.
Jingyue Wu12b0c282015-06-15 05:46:29 +00001642///
1643/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
1644/// we cannot optimize based on the assumption that it is zero without changing
1645/// it to be an explicit zero. If we don't change it to zero, other code could
1646/// optimized based on the contradictory assumption that it is non-zero.
1647/// Because instcombine aggressively folds operations with undef args anyway,
1648/// this won't lose us code quality.
1649///
1650/// This function is defined on values with integer type, values with pointer
1651/// type, and vectors of integers. In the case
1652/// where V is a vector, known zero, and known one values are the
1653/// same width as the vector element, and the bit is set only if it is true
1654/// for all of the elements in the vector.
Craig Topperb45eabc2017-04-26 16:39:58 +00001655void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth,
1656 const Query &Q) {
Jingyue Wu12b0c282015-06-15 05:46:29 +00001657 assert(V && "No Value?");
1658 assert(Depth <= MaxDepth && "Limit Search Depth");
Craig Topperb45eabc2017-04-26 16:39:58 +00001659 unsigned BitWidth = Known.getBitWidth();
Jingyue Wu12b0c282015-06-15 05:46:29 +00001660
Craig Topperfde47232017-07-09 07:04:03 +00001661 assert((V->getType()->isIntOrIntVectorTy(BitWidth) ||
Craig Topper95d23472017-07-09 07:04:00 +00001662 V->getType()->isPtrOrPtrVectorTy()) &&
Sanjay Pateldba8b4c2016-06-02 20:01:37 +00001663 "Not integer or pointer type!");
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00001664
1665 Type *ScalarTy = V->getType()->getScalarType();
1666 unsigned ExpectedWidth = ScalarTy->isPointerTy() ?
Nicola Zaghenf798eb22019-12-12 10:25:14 +00001667 Q.DL.getIndexTypeSizeInBits(ScalarTy) : Q.DL.getTypeSizeInBits(ScalarTy);
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00001668 assert(ExpectedWidth == BitWidth && "V and Known should have same BitWidth");
Craig Topperd73c6b42017-03-23 07:06:39 +00001669 (void)BitWidth;
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00001670 (void)ExpectedWidth;
Jingyue Wu12b0c282015-06-15 05:46:29 +00001671
Sanjay Patelc96f6db2016-09-16 21:20:36 +00001672 const APInt *C;
1673 if (match(V, m_APInt(C))) {
1674 // We know all of the bits for a scalar constant or a splat vector constant!
Craig Topperb45eabc2017-04-26 16:39:58 +00001675 Known.One = *C;
1676 Known.Zero = ~Known.One;
Jingyue Wu12b0c282015-06-15 05:46:29 +00001677 return;
1678 }
1679 // Null and aggregate-zero are all-zeros.
Sanjay Patele8dc0902016-05-23 17:57:54 +00001680 if (isa<ConstantPointerNull>(V) || isa<ConstantAggregateZero>(V)) {
Craig Topperf0aeee02017-05-05 17:36:09 +00001681 Known.setAllZero();
Jingyue Wu12b0c282015-06-15 05:46:29 +00001682 return;
1683 }
1684 // Handle a constant vector by taking the intersection of the known bits of
David Majnemer3918cdd2016-05-04 06:13:33 +00001685 // each element.
Pete Cooper35b00d52016-08-13 01:05:32 +00001686 if (const ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(V)) {
Jingyue Wu12b0c282015-06-15 05:46:29 +00001687 // We know that CDS must be a vector of integers. Take the intersection of
1688 // each element.
Craig Topperb45eabc2017-04-26 16:39:58 +00001689 Known.Zero.setAllBits(); Known.One.setAllBits();
Jingyue Wu12b0c282015-06-15 05:46:29 +00001690 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
Craig Topperb98ee582017-10-21 16:35:39 +00001691 APInt Elt = CDS->getElementAsAPInt(i);
Craig Topperb45eabc2017-04-26 16:39:58 +00001692 Known.Zero &= ~Elt;
1693 Known.One &= Elt;
Jingyue Wu12b0c282015-06-15 05:46:29 +00001694 }
1695 return;
1696 }
1697
Pete Cooper35b00d52016-08-13 01:05:32 +00001698 if (const auto *CV = dyn_cast<ConstantVector>(V)) {
David Majnemer3918cdd2016-05-04 06:13:33 +00001699 // We know that CV must be a vector of integers. Take the intersection of
1700 // each element.
Craig Topperb45eabc2017-04-26 16:39:58 +00001701 Known.Zero.setAllBits(); Known.One.setAllBits();
David Majnemer3918cdd2016-05-04 06:13:33 +00001702 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
1703 Constant *Element = CV->getAggregateElement(i);
1704 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Element);
1705 if (!ElementCI) {
Craig Topperf0aeee02017-05-05 17:36:09 +00001706 Known.resetAll();
David Majnemer3918cdd2016-05-04 06:13:33 +00001707 return;
1708 }
Craig Topperb98ee582017-10-21 16:35:39 +00001709 const APInt &Elt = ElementCI->getValue();
Craig Topperb45eabc2017-04-26 16:39:58 +00001710 Known.Zero &= ~Elt;
1711 Known.One &= Elt;
David Majnemer3918cdd2016-05-04 06:13:33 +00001712 }
1713 return;
1714 }
1715
Jingyue Wu12b0c282015-06-15 05:46:29 +00001716 // Start out not knowing anything.
Craig Topperf0aeee02017-05-05 17:36:09 +00001717 Known.resetAll();
Jingyue Wu12b0c282015-06-15 05:46:29 +00001718
Duncan P. N. Exon Smithb1b208a2016-09-24 20:42:02 +00001719 // We can't imply anything about undefs.
1720 if (isa<UndefValue>(V))
1721 return;
1722
1723 // There's no point in looking through other users of ConstantData for
1724 // assumptions. Confirm that we've handled them all.
1725 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
1726
Jingyue Wu12b0c282015-06-15 05:46:29 +00001727 // Limit search depth.
1728 // All recursive calls that increase depth must come after this.
1729 if (Depth == MaxDepth)
1730 return;
1731
1732 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
1733 // the bits of its aliasee.
Pete Cooper35b00d52016-08-13 01:05:32 +00001734 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
Sanjoy Das5ce32722016-04-08 00:48:30 +00001735 if (!GA->isInterposable())
Craig Topperb45eabc2017-04-26 16:39:58 +00001736 computeKnownBits(GA->getAliasee(), Known, Depth + 1, Q);
Jingyue Wu12b0c282015-06-15 05:46:29 +00001737 return;
1738 }
1739
Pete Cooper35b00d52016-08-13 01:05:32 +00001740 if (const Operator *I = dyn_cast<Operator>(V))
Craig Topperb45eabc2017-04-26 16:39:58 +00001741 computeKnownBitsFromOperator(I, Known, Depth, Q);
Sanjay Patela67559c2015-09-25 20:12:43 +00001742
Craig Topperb45eabc2017-04-26 16:39:58 +00001743 // Aligned pointers have trailing zeros - refine Known.Zero set
Artur Pilipenko029d8532015-09-30 11:55:45 +00001744 if (V->getType()->isPointerTy()) {
Guillaume Chateletbae629b2019-10-15 13:58:22 +00001745 const MaybeAlign Align = V->getPointerAlignment(Q.DL);
Artur Pilipenko029d8532015-09-30 11:55:45 +00001746 if (Align)
Guillaume Chateletbae629b2019-10-15 13:58:22 +00001747 Known.Zero.setLowBits(countTrailingZeros(Align->value()));
Artur Pilipenko029d8532015-09-30 11:55:45 +00001748 }
1749
Craig Topperb45eabc2017-04-26 16:39:58 +00001750 // computeKnownBitsFromAssume strictly refines Known.
1751 // Therefore, we run them after computeKnownBitsFromOperator.
Jingyue Wu12b0c282015-06-15 05:46:29 +00001752
1753 // Check whether a nearby assume intrinsic can determine some known bits.
Craig Topperb45eabc2017-04-26 16:39:58 +00001754 computeKnownBitsFromAssume(V, Known, Depth, Q);
Jingyue Wu12b0c282015-06-15 05:46:29 +00001755
Craig Topperb45eabc2017-04-26 16:39:58 +00001756 assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?");
Chris Lattner965c7692008-06-02 01:18:21 +00001757}
1758
Sanjay Patelaee84212014-11-04 16:27:42 +00001759/// Return true if the given value is known to have exactly one
Duncan Sandsd3951082011-01-25 09:38:29 +00001760/// bit set when defined. For vectors return true if every element is known to
Sanjay Patelaee84212014-11-04 16:27:42 +00001761/// be a power of two when defined. Supports values with integer or pointer
Duncan Sandsd3951082011-01-25 09:38:29 +00001762/// types and vectors of integers.
Pete Cooper35b00d52016-08-13 01:05:32 +00001763bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001764 const Query &Q) {
Craig Topper7227eba2017-08-21 22:56:12 +00001765 assert(Depth <= MaxDepth && "Limit Search Depth");
1766
Simon Pilgrim9f2ae7e2018-02-06 18:39:23 +00001767 // Attempt to match against constants.
1768 if (OrZero && match(V, m_Power2OrZero()))
1769 return true;
1770 if (match(V, m_Power2()))
1771 return true;
Duncan Sandsd3951082011-01-25 09:38:29 +00001772
1773 // 1 << X is clearly a power of two if the one is not shifted off the end. If
1774 // it is shifted off the end then the result is undefined.
1775 if (match(V, m_Shl(m_One(), m_Value())))
1776 return true;
1777
Craig Topperbcfd2d12017-04-20 16:56:25 +00001778 // (signmask) >>l X is clearly a power of two if the one is not shifted off
1779 // the bottom. If it is shifted off the bottom then the result is undefined.
1780 if (match(V, m_LShr(m_SignMask(), m_Value())))
Duncan Sandsd3951082011-01-25 09:38:29 +00001781 return true;
1782
1783 // The remaining tests are all recursive, so bail out if we hit the limit.
1784 if (Depth++ == MaxDepth)
1785 return false;
1786
Craig Topper9f008862014-04-15 04:59:12 +00001787 Value *X = nullptr, *Y = nullptr;
Sanjay Patel41160c22015-12-30 22:40:52 +00001788 // A shift left or a logical shift right of a power of two is a power of two
1789 // or zero.
Duncan Sands985ba632011-10-28 18:30:05 +00001790 if (OrZero && (match(V, m_Shl(m_Value(X), m_Value())) ||
Sanjay Patel41160c22015-12-30 22:40:52 +00001791 match(V, m_LShr(m_Value(X), m_Value()))))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001792 return isKnownToBeAPowerOfTwo(X, /*OrZero*/ true, Depth, Q);
Duncan Sands985ba632011-10-28 18:30:05 +00001793
Pete Cooper35b00d52016-08-13 01:05:32 +00001794 if (const ZExtInst *ZI = dyn_cast<ZExtInst>(V))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001795 return isKnownToBeAPowerOfTwo(ZI->getOperand(0), OrZero, Depth, Q);
Duncan Sandsd3951082011-01-25 09:38:29 +00001796
Pete Cooper35b00d52016-08-13 01:05:32 +00001797 if (const SelectInst *SI = dyn_cast<SelectInst>(V))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001798 return isKnownToBeAPowerOfTwo(SI->getTrueValue(), OrZero, Depth, Q) &&
1799 isKnownToBeAPowerOfTwo(SI->getFalseValue(), OrZero, Depth, Q);
Duncan Sandsba286d72011-10-26 20:55:21 +00001800
Duncan Sandsba286d72011-10-26 20:55:21 +00001801 if (OrZero && match(V, m_And(m_Value(X), m_Value(Y)))) {
1802 // A power of two and'd with anything is a power of two or zero.
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001803 if (isKnownToBeAPowerOfTwo(X, /*OrZero*/ true, Depth, Q) ||
1804 isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, Depth, Q))
Duncan Sandsba286d72011-10-26 20:55:21 +00001805 return true;
1806 // X & (-X) is always a power of two or zero.
1807 if (match(X, m_Neg(m_Specific(Y))) || match(Y, m_Neg(m_Specific(X))))
1808 return true;
1809 return false;
1810 }
Duncan Sandsd3951082011-01-25 09:38:29 +00001811
David Majnemerb7d54092013-07-30 21:01:36 +00001812 // Adding a power-of-two or zero to the same power-of-two or zero yields
1813 // either the original power-of-two, a larger power-of-two or zero.
1814 if (match(V, m_Add(m_Value(X), m_Value(Y)))) {
Pete Cooper35b00d52016-08-13 01:05:32 +00001815 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(V);
Florian Hahn19f9e322018-08-17 14:39:04 +00001816 if (OrZero || Q.IIQ.hasNoUnsignedWrap(VOBO) ||
1817 Q.IIQ.hasNoSignedWrap(VOBO)) {
David Majnemerb7d54092013-07-30 21:01:36 +00001818 if (match(X, m_And(m_Specific(Y), m_Value())) ||
1819 match(X, m_And(m_Value(), m_Specific(Y))))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001820 if (isKnownToBeAPowerOfTwo(Y, OrZero, Depth, Q))
David Majnemerb7d54092013-07-30 21:01:36 +00001821 return true;
1822 if (match(Y, m_And(m_Specific(X), m_Value())) ||
1823 match(Y, m_And(m_Value(), m_Specific(X))))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001824 if (isKnownToBeAPowerOfTwo(X, OrZero, Depth, Q))
David Majnemerb7d54092013-07-30 21:01:36 +00001825 return true;
1826
1827 unsigned BitWidth = V->getType()->getScalarSizeInBits();
Craig Topperb45eabc2017-04-26 16:39:58 +00001828 KnownBits LHSBits(BitWidth);
1829 computeKnownBits(X, LHSBits, Depth, Q);
David Majnemerb7d54092013-07-30 21:01:36 +00001830
Craig Topperb45eabc2017-04-26 16:39:58 +00001831 KnownBits RHSBits(BitWidth);
1832 computeKnownBits(Y, RHSBits, Depth, Q);
David Majnemerb7d54092013-07-30 21:01:36 +00001833 // If i8 V is a power of two or zero:
1834 // ZeroBits: 1 1 1 0 1 1 1 1
1835 // ~ZeroBits: 0 0 0 1 0 0 0 0
Craig Topperb45eabc2017-04-26 16:39:58 +00001836 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
David Majnemerb7d54092013-07-30 21:01:36 +00001837 // If OrZero isn't set, we cannot give back a zero result.
1838 // Make sure either the LHS or RHS has a bit set.
Craig Topperb45eabc2017-04-26 16:39:58 +00001839 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
David Majnemerb7d54092013-07-30 21:01:36 +00001840 return true;
1841 }
1842 }
David Majnemerbeab5672013-05-18 19:30:37 +00001843
Nick Lewyckyc9aab852011-02-28 08:02:21 +00001844 // An exact divide or right shift can only shift off zero bits, so the result
Nick Lewyckyf0469af2011-03-21 21:40:32 +00001845 // is a power of two only if the first operand is a power of two and not
1846 // copying a sign bit (sdiv int_min, 2).
Benjamin Kramer9442cd02012-01-01 17:55:30 +00001847 if (match(V, m_Exact(m_LShr(m_Value(), m_Value()))) ||
1848 match(V, m_Exact(m_UDiv(m_Value(), m_Value())))) {
Hal Finkel60db0582014-09-07 18:57:58 +00001849 return isKnownToBeAPowerOfTwo(cast<Operator>(V)->getOperand(0), OrZero,
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001850 Depth, Q);
Nick Lewyckyc9aab852011-02-28 08:02:21 +00001851 }
1852
Duncan Sandsd3951082011-01-25 09:38:29 +00001853 return false;
1854}
1855
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00001856/// Test whether a GEP's result is known to be non-null.
Chandler Carruth80d3e562012-12-07 02:08:58 +00001857///
1858/// Uses properties inherent in a GEP to try to determine whether it is known
1859/// to be non-null.
1860///
1861/// Currently this routine does not support vector GEPs.
Pete Cooper35b00d52016-08-13 01:05:32 +00001862static bool isGEPKnownNonNull(const GEPOperator *GEP, unsigned Depth,
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001863 const Query &Q) {
Manoj Gupta77eeac32018-07-09 22:27:23 +00001864 const Function *F = nullptr;
1865 if (const Instruction *I = dyn_cast<Instruction>(GEP))
1866 F = I->getFunction();
1867
1868 if (!GEP->isInBounds() ||
1869 NullPointerIsDefined(F, GEP->getPointerAddressSpace()))
Chandler Carruth80d3e562012-12-07 02:08:58 +00001870 return false;
1871
1872 // FIXME: Support vector-GEPs.
1873 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
1874
1875 // If the base pointer is non-null, we cannot walk to a null address with an
1876 // inbounds GEP in address space zero.
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001877 if (isKnownNonZero(GEP->getPointerOperand(), Depth, Q))
Chandler Carruth80d3e562012-12-07 02:08:58 +00001878 return true;
1879
Chandler Carruth80d3e562012-12-07 02:08:58 +00001880 // Walk the GEP operands and see if any operand introduces a non-zero offset.
1881 // If so, then the GEP cannot produce a null pointer, as doing so would
1882 // inherently violate the inbounds contract within address space zero.
1883 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
1884 GTI != GTE; ++GTI) {
1885 // Struct types are easy -- they must always be indexed by a constant.
Peter Collingbourneab85225b2016-12-02 02:24:42 +00001886 if (StructType *STy = GTI.getStructTypeOrNull()) {
Chandler Carruth80d3e562012-12-07 02:08:58 +00001887 ConstantInt *OpC = cast<ConstantInt>(GTI.getOperand());
1888 unsigned ElementIdx = OpC->getZExtValue();
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001889 const StructLayout *SL = Q.DL.getStructLayout(STy);
Chandler Carruth80d3e562012-12-07 02:08:58 +00001890 uint64_t ElementOffset = SL->getElementOffset(ElementIdx);
1891 if (ElementOffset > 0)
1892 return true;
1893 continue;
1894 }
1895
1896 // If we have a zero-sized type, the index doesn't matter. Keep looping.
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001897 if (Q.DL.getTypeAllocSize(GTI.getIndexedType()) == 0)
Chandler Carruth80d3e562012-12-07 02:08:58 +00001898 continue;
1899
1900 // Fast path the constant operand case both for efficiency and so we don't
1901 // increment Depth when just zipping down an all-constant GEP.
1902 if (ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand())) {
1903 if (!OpC->isZero())
1904 return true;
1905 continue;
1906 }
1907
1908 // We post-increment Depth here because while isKnownNonZero increments it
1909 // as well, when we pop back up that increment won't persist. We don't want
1910 // to recurse 10k times just because we have 10k GEP operands. We don't
1911 // bail completely out because we want to handle constant GEPs regardless
1912 // of depth.
1913 if (Depth++ >= MaxDepth)
1914 continue;
1915
Matthias Braunfeb81bc2016-01-15 22:22:04 +00001916 if (isKnownNonZero(GTI.getOperand(), Depth, Q))
Chandler Carruth80d3e562012-12-07 02:08:58 +00001917 return true;
1918 }
1919
1920 return false;
1921}
1922
Nikita Popov9fd02a72019-05-08 14:50:01 +00001923static bool isKnownNonNullFromDominatingCondition(const Value *V,
Nuno Lopes404f1062017-09-09 18:23:11 +00001924 const Instruction *CtxI,
1925 const DominatorTree *DT) {
Johannes Doerferta7d992c2019-12-08 22:11:16 -06001926 if (isa<Constant>(V))
1927 return false;
Nuno Lopes404f1062017-09-09 18:23:11 +00001928
1929 if (!CtxI || !DT)
1930 return false;
1931
1932 unsigned NumUsesExplored = 0;
1933 for (auto *U : V->users()) {
1934 // Avoid massive lists
1935 if (NumUsesExplored >= DomConditionsMaxUses)
1936 break;
1937 NumUsesExplored++;
1938
Nikita Popov9fd02a72019-05-08 14:50:01 +00001939 // If the value is used as an argument to a call or invoke, then argument
1940 // attributes may provide an answer about null-ness.
1941 if (auto CS = ImmutableCallSite(U))
1942 if (auto *CalledFunc = CS.getCalledFunction())
1943 for (const Argument &Arg : CalledFunc->args())
1944 if (CS.getArgOperand(Arg.getArgNo()) == V &&
1945 Arg.hasNonNullAttr() && DT->dominates(CS.getInstruction(), CtxI))
1946 return true;
Nuno Lopes404f1062017-09-09 18:23:11 +00001947
Danila Kutenin19e83a92019-12-11 20:30:54 +01001948 // If the value is used as a load/store, then the pointer must be non null.
1949 if (V == getLoadStorePointerOperand(U)) {
1950 const Instruction *I = cast<Instruction>(U);
1951 if (!NullPointerIsDefined(I->getFunction(),
1952 V->getType()->getPointerAddressSpace()) &&
1953 DT->dominates(I, CtxI))
1954 return true;
1955 }
1956
Nuno Lopes404f1062017-09-09 18:23:11 +00001957 // Consider only compare instructions uniquely controlling a branch
1958 CmpInst::Predicate Pred;
1959 if (!match(const_cast<User *>(U),
1960 m_c_ICmp(Pred, m_Specific(V), m_Zero())) ||
1961 (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE))
1962 continue;
1963
Max Kazantsev2dbbd642018-08-06 11:14:18 +00001964 SmallVector<const User *, 4> WorkList;
1965 SmallPtrSet<const User *, 4> Visited;
Nuno Lopes404f1062017-09-09 18:23:11 +00001966 for (auto *CmpU : U->users()) {
Max Kazantsev2dbbd642018-08-06 11:14:18 +00001967 assert(WorkList.empty() && "Should be!");
1968 if (Visited.insert(CmpU).second)
1969 WorkList.push_back(CmpU);
Nuno Lopes404f1062017-09-09 18:23:11 +00001970
Max Kazantsev2dbbd642018-08-06 11:14:18 +00001971 while (!WorkList.empty()) {
1972 auto *Curr = WorkList.pop_back_val();
1973
1974 // If a user is an AND, add all its users to the work list. We only
1975 // propagate "pred != null" condition through AND because it is only
1976 // correct to assume that all conditions of AND are met in true branch.
1977 // TODO: Support similar logic of OR and EQ predicate?
1978 if (Pred == ICmpInst::ICMP_NE)
1979 if (auto *BO = dyn_cast<BinaryOperator>(Curr))
1980 if (BO->getOpcode() == Instruction::And) {
1981 for (auto *BOU : BO->users())
1982 if (Visited.insert(BOU).second)
1983 WorkList.push_back(BOU);
1984 continue;
1985 }
1986
1987 if (const BranchInst *BI = dyn_cast<BranchInst>(Curr)) {
1988 assert(BI->isConditional() && "uses a comparison!");
1989
1990 BasicBlock *NonNullSuccessor =
1991 BI->getSuccessor(Pred == ICmpInst::ICMP_EQ ? 1 : 0);
1992 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
1993 if (Edge.isSingleEdge() && DT->dominates(Edge, CtxI->getParent()))
1994 return true;
Max Kazantsev3c284bd2018-08-30 03:39:16 +00001995 } else if (Pred == ICmpInst::ICMP_NE && isGuard(Curr) &&
Max Kazantsev2dbbd642018-08-06 11:14:18 +00001996 DT->dominates(cast<Instruction>(Curr), CtxI)) {
Nuno Lopes404f1062017-09-09 18:23:11 +00001997 return true;
Max Kazantsev2dbbd642018-08-06 11:14:18 +00001998 }
Nuno Lopes404f1062017-09-09 18:23:11 +00001999 }
2000 }
2001 }
2002
2003 return false;
2004}
2005
Philip Reames4cb4d3e2014-10-30 20:25:19 +00002006/// Does the 'Range' metadata (which must be a valid MD_range operand list)
2007/// ensure that the value it's attached to is never Value? 'RangeType' is
2008/// is the type of the value described by the range.
Pete Cooper35b00d52016-08-13 01:05:32 +00002009static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
Philip Reames4cb4d3e2014-10-30 20:25:19 +00002010 const unsigned NumRanges = Ranges->getNumOperands() / 2;
2011 assert(NumRanges >= 1);
2012 for (unsigned i = 0; i < NumRanges; ++i) {
Duncan P. N. Exon Smith5bf8fef2014-12-09 18:38:53 +00002013 ConstantInt *Lower =
2014 mdconst::extract<ConstantInt>(Ranges->getOperand(2 * i + 0));
2015 ConstantInt *Upper =
2016 mdconst::extract<ConstantInt>(Ranges->getOperand(2 * i + 1));
Philip Reames4cb4d3e2014-10-30 20:25:19 +00002017 ConstantRange Range(Lower->getValue(), Upper->getValue());
2018 if (Range.contains(Value))
2019 return false;
2020 }
2021 return true;
2022}
2023
Sanjay Patel97e4b9872017-02-12 15:35:34 +00002024/// Return true if the given value is known to be non-zero when defined. For
2025/// vectors, return true if every element is known to be non-zero when
2026/// defined. For pointers, if the context instruction and dominator tree are
2027/// specified, perform context-sensitive analysis and return true if the
2028/// pointer couldn't possibly be null at the specified instruction.
2029/// Supports values with integer or pointer type and vectors of integers.
Pete Cooper35b00d52016-08-13 01:05:32 +00002030bool isKnownNonZero(const Value *V, unsigned Depth, const Query &Q) {
Sanjay Patel8ec7e7c2016-05-22 16:07:20 +00002031 if (auto *C = dyn_cast<Constant>(V)) {
Duncan Sandsd3951082011-01-25 09:38:29 +00002032 if (C->isNullValue())
2033 return false;
2034 if (isa<ConstantInt>(C))
2035 // Must be non-zero due to null test above.
2036 return true;
Sanjay Patel23019d12016-05-24 14:18:49 +00002037
Johannes Doerfert2d63fbb2019-07-15 03:24:35 +00002038 if (auto *CE = dyn_cast<ConstantExpr>(C)) {
2039 // See the comment for IntToPtr/PtrToInt instructions below.
2040 if (CE->getOpcode() == Instruction::IntToPtr ||
2041 CE->getOpcode() == Instruction::PtrToInt)
2042 if (Q.DL.getTypeSizeInBits(CE->getOperand(0)->getType()) <=
2043 Q.DL.getTypeSizeInBits(CE->getType()))
2044 return isKnownNonZero(CE->getOperand(0), Depth, Q);
2045 }
2046
Sanjay Patel23019d12016-05-24 14:18:49 +00002047 // For constant vectors, check that all elements are undefined or known
2048 // non-zero to determine that the whole vector is known non-zero.
2049 if (auto *VecTy = dyn_cast<VectorType>(C->getType())) {
2050 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
2051 Constant *Elt = C->getAggregateElement(i);
2052 if (!Elt || Elt->isNullValue())
2053 return false;
2054 if (!isa<UndefValue>(Elt) && !isa<ConstantInt>(Elt))
2055 return false;
2056 }
2057 return true;
2058 }
2059
Nuno Lopes404f1062017-09-09 18:23:11 +00002060 // A global variable in address space 0 is non null unless extern weak
2061 // or an absolute symbol reference. Other address spaces may have null as a
2062 // valid address for a global, so we can't assume anything.
2063 if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
2064 if (!GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
2065 GV->getType()->getAddressSpace() == 0)
2066 return true;
2067 } else
2068 return false;
Duncan Sandsd3951082011-01-25 09:38:29 +00002069 }
2070
Sanjay Patel8ec7e7c2016-05-22 16:07:20 +00002071 if (auto *I = dyn_cast<Instruction>(V)) {
Florian Hahn19f9e322018-08-17 14:39:04 +00002072 if (MDNode *Ranges = Q.IIQ.getMetadata(I, LLVMContext::MD_range)) {
Philip Reames4cb4d3e2014-10-30 20:25:19 +00002073 // If the possible ranges don't contain zero, then the value is
2074 // definitely non-zero.
Sanjay Patel8ec7e7c2016-05-22 16:07:20 +00002075 if (auto *Ty = dyn_cast<IntegerType>(V->getType())) {
Philip Reames4cb4d3e2014-10-30 20:25:19 +00002076 const APInt ZeroValue(Ty->getBitWidth(), 0);
2077 if (rangeMetadataExcludesValue(Ranges, ZeroValue))
2078 return true;
2079 }
2080 }
2081 }
2082
Karl-Johan Karlssonebaaa2d2018-05-30 15:56:46 +00002083 // Some of the tests below are recursive, so bail out if we hit the limit.
2084 if (Depth++ >= MaxDepth)
2085 return false;
2086
Nuno Lopes404f1062017-09-09 18:23:11 +00002087 // Check for pointer simplifications.
2088 if (V->getType()->isPointerTy()) {
2089 // Alloca never returns null, malloc might.
2090 if (isa<AllocaInst>(V) && Q.DL.getAllocaAddrSpace() == 0)
2091 return true;
2092
2093 // A byval, inalloca, or nonnull argument is never null.
2094 if (const Argument *A = dyn_cast<Argument>(V))
2095 if (A->hasByValOrInAllocaAttr() || A->hasNonNullAttr())
2096 return true;
2097
2098 // A Load tagged with nonnull metadata is never null.
2099 if (const LoadInst *LI = dyn_cast<LoadInst>(V))
Florian Hahn19f9e322018-08-17 14:39:04 +00002100 if (Q.IIQ.getMetadata(LI, LLVMContext::MD_nonnull))
Nuno Lopes404f1062017-09-09 18:23:11 +00002101 return true;
2102
Chandler Carruth363ac682019-01-07 05:42:51 +00002103 if (const auto *Call = dyn_cast<CallBase>(V)) {
2104 if (Call->isReturnNonNull())
Nuno Lopes404f1062017-09-09 18:23:11 +00002105 return true;
Florian Hahnfd72bf22019-08-15 12:13:02 +00002106 if (const auto *RP = getArgumentAliasingToReturnedPointer(Call, true))
Karl-Johan Karlssonebaaa2d2018-05-30 15:56:46 +00002107 return isKnownNonZero(RP, Depth, Q);
Piotr Padlewski5642a422018-05-18 23:54:33 +00002108 }
Nuno Lopes404f1062017-09-09 18:23:11 +00002109 }
2110
Johannes Doerferta7d992c2019-12-08 22:11:16 -06002111 if (isKnownNonNullFromDominatingCondition(V, Q.CxtI, Q.DT))
2112 return true;
Duncan Sandsd3951082011-01-25 09:38:29 +00002113
Nuno Lopes404f1062017-09-09 18:23:11 +00002114 // Check for recursive pointer simplifications.
Chandler Carruth80d3e562012-12-07 02:08:58 +00002115 if (V->getType()->isPointerTy()) {
Johannes Doerfert00102c72019-01-26 23:40:35 +00002116 // Look through bitcast operations, GEPs, and int2ptr instructions as they
2117 // do not alter the value, or at least not the nullness property of the
2118 // value, e.g., int2ptr is allowed to zero/sign extend the value.
2119 //
2120 // Note that we have to take special care to avoid looking through
2121 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
2122 // as casts that can alter the value, e.g., AddrSpaceCasts.
Pete Cooper35b00d52016-08-13 01:05:32 +00002123 if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002124 if (isGEPKnownNonNull(GEP, Depth, Q))
Chandler Carruth80d3e562012-12-07 02:08:58 +00002125 return true;
Johannes Doerfert00102c72019-01-26 23:40:35 +00002126
2127 if (auto *BCO = dyn_cast<BitCastOperator>(V))
2128 return isKnownNonZero(BCO->getOperand(0), Depth, Q);
2129
2130 if (auto *I2P = dyn_cast<IntToPtrInst>(V))
2131 if (Q.DL.getTypeSizeInBits(I2P->getSrcTy()) <=
2132 Q.DL.getTypeSizeInBits(I2P->getDestTy()))
2133 return isKnownNonZero(I2P->getOperand(0), Depth, Q);
Chandler Carruth80d3e562012-12-07 02:08:58 +00002134 }
2135
Johannes Doerfert00102c72019-01-26 23:40:35 +00002136 // Similar to int2ptr above, we can look through ptr2int here if the cast
2137 // is a no-op or an extend and not a truncate.
2138 if (auto *P2I = dyn_cast<PtrToIntInst>(V))
2139 if (Q.DL.getTypeSizeInBits(P2I->getSrcTy()) <=
2140 Q.DL.getTypeSizeInBits(P2I->getDestTy()))
2141 return isKnownNonZero(P2I->getOperand(0), Depth, Q);
2142
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002143 unsigned BitWidth = getBitWidth(V->getType()->getScalarType(), Q.DL);
Duncan Sandsd3951082011-01-25 09:38:29 +00002144
2145 // X | Y != 0 if X != 0 or Y != 0.
Craig Topper9f008862014-04-15 04:59:12 +00002146 Value *X = nullptr, *Y = nullptr;
Duncan Sandsd3951082011-01-25 09:38:29 +00002147 if (match(V, m_Or(m_Value(X), m_Value(Y))))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002148 return isKnownNonZero(X, Depth, Q) || isKnownNonZero(Y, Depth, Q);
Duncan Sandsd3951082011-01-25 09:38:29 +00002149
2150 // ext X != 0 if X != 0.
2151 if (isa<SExtInst>(V) || isa<ZExtInst>(V))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002152 return isKnownNonZero(cast<Instruction>(V)->getOperand(0), Depth, Q);
Duncan Sandsd3951082011-01-25 09:38:29 +00002153
Duncan Sands2e9e4f12011-01-29 13:27:00 +00002154 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
Duncan Sandsd3951082011-01-25 09:38:29 +00002155 // if the lowest bit is shifted off the end.
Craig Topper6b3940a2017-05-03 22:25:19 +00002156 if (match(V, m_Shl(m_Value(X), m_Value(Y)))) {
Nick Lewyckyc9aab852011-02-28 08:02:21 +00002157 // shl nuw can't remove any non-zero bits.
Pete Cooper35b00d52016-08-13 01:05:32 +00002158 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(V);
Florian Hahn19f9e322018-08-17 14:39:04 +00002159 if (Q.IIQ.hasNoUnsignedWrap(BO))
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002160 return isKnownNonZero(X, Depth, Q);
Nick Lewyckyc9aab852011-02-28 08:02:21 +00002161
Craig Topperb45eabc2017-04-26 16:39:58 +00002162 KnownBits Known(BitWidth);
2163 computeKnownBits(X, Known, Depth, Q);
2164 if (Known.One[0])
Duncan Sandsd3951082011-01-25 09:38:29 +00002165 return true;
2166 }
Duncan Sands2e9e4f12011-01-29 13:27:00 +00002167 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
Duncan Sandsd3951082011-01-25 09:38:29 +00002168 // defined if the sign bit is shifted off the end.
2169 else if (match(V, m_Shr(m_Value(X), m_Value(Y)))) {
Nick Lewyckyc9aab852011-02-28 08:02:21 +00002170 // shr exact can only shift out zero bits.
Pete Cooper35b00d52016-08-13 01:05:32 +00002171 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(V);
Nick Lewyckyc9aab852011-02-28 08:02:21 +00002172 if (BO->isExact())
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002173 return isKnownNonZero(X, Depth, Q);
Nick Lewyckyc9aab852011-02-28 08:02:21 +00002174
Craig Topper6e11a052017-05-08 16:22:48 +00002175 KnownBits Known = computeKnownBits(X, Depth, Q);
2176 if (Known.isNegative())
Duncan Sandsd3951082011-01-25 09:38:29 +00002177 return true;
James Molloyb6be1eb2015-09-24 16:06:32 +00002178
2179 // If the shifter operand is a constant, and all of the bits shifted
2180 // out are known to be zero, and X is known non-zero then at least one
2181 // non-zero bit must remain.
2182 if (ConstantInt *Shift = dyn_cast<ConstantInt>(Y)) {
James Molloyb6be1eb2015-09-24 16:06:32 +00002183 auto ShiftVal = Shift->getLimitedValue(BitWidth - 1);
2184 // Is there a known one in the portion not shifted out?
Craig Topper8df66c62017-05-12 17:20:30 +00002185 if (Known.countMaxLeadingZeros() < BitWidth - ShiftVal)
James Molloyb6be1eb2015-09-24 16:06:32 +00002186 return true;
2187 // Are all the bits to be shifted out known zero?
NAKAMURA Takumi76bab1f2017-07-11 02:31:51 +00002188 if (Known.countMinTrailingZeros() >= ShiftVal)
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002189 return isKnownNonZero(X, Depth, Q);
James Molloyb6be1eb2015-09-24 16:06:32 +00002190 }
Duncan Sandsd3951082011-01-25 09:38:29 +00002191 }
Nick Lewyckyc9aab852011-02-28 08:02:21 +00002192 // div exact can only produce a zero if the dividend is zero.
Benjamin Kramer9442cd02012-01-01 17:55:30 +00002193 else if (match(V, m_Exact(m_IDiv(m_Value(X), m_Value())))) {
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002194 return isKnownNonZero(X, Depth, Q);
Nick Lewyckyc9aab852011-02-28 08:02:21 +00002195 }
Duncan Sandsd3951082011-01-25 09:38:29 +00002196 // X + Y.
2197 else if (match(V, m_Add(m_Value(X), m_Value(Y)))) {
Craig Topper6e11a052017-05-08 16:22:48 +00002198 KnownBits XKnown = computeKnownBits(X, Depth, Q);
2199 KnownBits YKnown = computeKnownBits(Y, Depth, Q);
Duncan Sandsd3951082011-01-25 09:38:29 +00002200
2201 // If X and Y are both non-negative (as signed values) then their sum is not
Duncan Sands9e9d5b22011-01-25 15:14:15 +00002202 // zero unless both X and Y are zero.
Craig Topper6e11a052017-05-08 16:22:48 +00002203 if (XKnown.isNonNegative() && YKnown.isNonNegative())
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002204 if (isKnownNonZero(X, Depth, Q) || isKnownNonZero(Y, Depth, Q))
Duncan Sands9e9d5b22011-01-25 15:14:15 +00002205 return true;
Duncan Sandsd3951082011-01-25 09:38:29 +00002206
2207 // If X and Y are both negative (as signed values) then their sum is not
2208 // zero unless both X and Y equal INT_MIN.
Craig Topper6e11a052017-05-08 16:22:48 +00002209 if (XKnown.isNegative() && YKnown.isNegative()) {
Duncan Sandsd3951082011-01-25 09:38:29 +00002210 APInt Mask = APInt::getSignedMaxValue(BitWidth);
2211 // The sign bit of X is set. If some other bit is set then X is not equal
2212 // to INT_MIN.
Craig Topper6e11a052017-05-08 16:22:48 +00002213 if (XKnown.One.intersects(Mask))
Duncan Sandsd3951082011-01-25 09:38:29 +00002214 return true;
2215 // The sign bit of Y is set. If some other bit is set then Y is not equal
2216 // to INT_MIN.
Craig Topper6e11a052017-05-08 16:22:48 +00002217 if (YKnown.One.intersects(Mask))
Duncan Sandsd3951082011-01-25 09:38:29 +00002218 return true;
2219 }
2220
2221 // The sum of a non-negative number and a power of two is not zero.
Craig Topper6e11a052017-05-08 16:22:48 +00002222 if (XKnown.isNonNegative() &&
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002223 isKnownToBeAPowerOfTwo(Y, /*OrZero*/ false, Depth, Q))
Duncan Sandsd3951082011-01-25 09:38:29 +00002224 return true;
Craig Topper6e11a052017-05-08 16:22:48 +00002225 if (YKnown.isNonNegative() &&
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002226 isKnownToBeAPowerOfTwo(X, /*OrZero*/ false, Depth, Q))
Duncan Sandsd3951082011-01-25 09:38:29 +00002227 return true;
2228 }
Duncan Sands7cb61e52011-10-27 19:16:21 +00002229 // X * Y.
2230 else if (match(V, m_Mul(m_Value(X), m_Value(Y)))) {
Pete Cooper35b00d52016-08-13 01:05:32 +00002231 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(V);
Duncan Sands7cb61e52011-10-27 19:16:21 +00002232 // If X and Y are non-zero then so is X * Y as long as the multiplication
2233 // does not overflow.
Florian Hahn19f9e322018-08-17 14:39:04 +00002234 if ((Q.IIQ.hasNoSignedWrap(BO) || Q.IIQ.hasNoUnsignedWrap(BO)) &&
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002235 isKnownNonZero(X, Depth, Q) && isKnownNonZero(Y, Depth, Q))
Duncan Sands7cb61e52011-10-27 19:16:21 +00002236 return true;
2237 }
Duncan Sandsd3951082011-01-25 09:38:29 +00002238 // (C ? X : Y) != 0 if X != 0 and Y != 0.
Pete Cooper35b00d52016-08-13 01:05:32 +00002239 else if (const SelectInst *SI = dyn_cast<SelectInst>(V)) {
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002240 if (isKnownNonZero(SI->getTrueValue(), Depth, Q) &&
2241 isKnownNonZero(SI->getFalseValue(), Depth, Q))
Duncan Sandsd3951082011-01-25 09:38:29 +00002242 return true;
2243 }
James Molloy897048b2015-09-29 14:08:45 +00002244 // PHI
Pete Cooper35b00d52016-08-13 01:05:32 +00002245 else if (const PHINode *PN = dyn_cast<PHINode>(V)) {
James Molloy897048b2015-09-29 14:08:45 +00002246 // Try and detect a recurrence that monotonically increases from a
2247 // starting value, as these are common as induction variables.
2248 if (PN->getNumIncomingValues() == 2) {
2249 Value *Start = PN->getIncomingValue(0);
2250 Value *Induction = PN->getIncomingValue(1);
2251 if (isa<ConstantInt>(Induction) && !isa<ConstantInt>(Start))
2252 std::swap(Start, Induction);
2253 if (ConstantInt *C = dyn_cast<ConstantInt>(Start)) {
2254 if (!C->isZero() && !C->isNegative()) {
2255 ConstantInt *X;
Florian Hahn19f9e322018-08-17 14:39:04 +00002256 if (Q.IIQ.UseInstrInfo &&
2257 (match(Induction, m_NSWAdd(m_Specific(PN), m_ConstantInt(X))) ||
James Molloy897048b2015-09-29 14:08:45 +00002258 match(Induction, m_NUWAdd(m_Specific(PN), m_ConstantInt(X)))) &&
2259 !X->isNegative())
2260 return true;
2261 }
2262 }
2263 }
Jun Bum Limca832662016-02-01 17:03:07 +00002264 // Check if all incoming values are non-zero constant.
Eugene Zelenko75075ef2017-09-01 21:37:29 +00002265 bool AllNonZeroConstants = llvm::all_of(PN->operands(), [](Value *V) {
Craig Topper79ab6432017-07-06 18:39:47 +00002266 return isa<ConstantInt>(V) && !cast<ConstantInt>(V)->isZero();
Jun Bum Limca832662016-02-01 17:03:07 +00002267 });
2268 if (AllNonZeroConstants)
2269 return true;
James Molloy897048b2015-09-29 14:08:45 +00002270 }
Duncan Sandsd3951082011-01-25 09:38:29 +00002271
Craig Topperb45eabc2017-04-26 16:39:58 +00002272 KnownBits Known(BitWidth);
2273 computeKnownBits(V, Known, Depth, Q);
2274 return Known.One != 0;
Duncan Sandsd3951082011-01-25 09:38:29 +00002275}
2276
James Molloy1d88d6f2015-10-22 13:18:42 +00002277/// Return true if V2 == V1 + X, where X is known non-zero.
Pete Cooper35b00d52016-08-13 01:05:32 +00002278static bool isAddOfNonZero(const Value *V1, const Value *V2, const Query &Q) {
2279 const BinaryOperator *BO = dyn_cast<BinaryOperator>(V1);
James Molloy1d88d6f2015-10-22 13:18:42 +00002280 if (!BO || BO->getOpcode() != Instruction::Add)
2281 return false;
2282 Value *Op = nullptr;
2283 if (V2 == BO->getOperand(0))
2284 Op = BO->getOperand(1);
2285 else if (V2 == BO->getOperand(1))
2286 Op = BO->getOperand(0);
2287 else
2288 return false;
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002289 return isKnownNonZero(Op, 0, Q);
James Molloy1d88d6f2015-10-22 13:18:42 +00002290}
2291
2292/// Return true if it is known that V1 != V2.
Pete Cooper35b00d52016-08-13 01:05:32 +00002293static bool isKnownNonEqual(const Value *V1, const Value *V2, const Query &Q) {
Craig Topper3002d5b2017-06-06 07:13:15 +00002294 if (V1 == V2)
James Molloy1d88d6f2015-10-22 13:18:42 +00002295 return false;
2296 if (V1->getType() != V2->getType())
2297 // We can't look through casts yet.
2298 return false;
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002299 if (isAddOfNonZero(V1, V2, Q) || isAddOfNonZero(V2, V1, Q))
James Molloy1d88d6f2015-10-22 13:18:42 +00002300 return true;
2301
Craig Topper3002d5b2017-06-06 07:13:15 +00002302 if (V1->getType()->isIntOrIntVectorTy()) {
James Molloy1d88d6f2015-10-22 13:18:42 +00002303 // Are any known bits in V1 contradictory to known bits in V2? If V1
2304 // has a known zero where V2 has a known one, they must not be equal.
Craig Topper8e662f72017-06-06 07:13:11 +00002305 KnownBits Known1 = computeKnownBits(V1, 0, Q);
2306 KnownBits Known2 = computeKnownBits(V2, 0, Q);
James Molloy1d88d6f2015-10-22 13:18:42 +00002307
Craig Topper8365df82017-06-06 07:13:09 +00002308 if (Known1.Zero.intersects(Known2.One) ||
2309 Known2.Zero.intersects(Known1.One))
James Molloy1d88d6f2015-10-22 13:18:42 +00002310 return true;
2311 }
2312 return false;
2313}
2314
Sanjay Patelaee84212014-11-04 16:27:42 +00002315/// Return true if 'V & Mask' is known to be zero. We use this predicate to
2316/// simplify operations downstream. Mask is known to be zero for bits that V
2317/// cannot have.
Chris Lattner4bc28252009-09-08 00:06:16 +00002318///
2319/// This function is defined on values with integer type, values with pointer
Mehdi Aminia28d91d2015-03-10 02:37:25 +00002320/// type, and vectors of integers. In the case
Chris Lattner4bc28252009-09-08 00:06:16 +00002321/// where V is a vector, the mask, known zero, and known one values are the
2322/// same width as the vector element, and the bit is set only if it is true
2323/// for all of the elements in the vector.
Pete Cooper35b00d52016-08-13 01:05:32 +00002324bool MaskedValueIsZero(const Value *V, const APInt &Mask, unsigned Depth,
Matthias Braunfeb81bc2016-01-15 22:22:04 +00002325 const Query &Q) {
Craig Topperb45eabc2017-04-26 16:39:58 +00002326 KnownBits Known(Mask.getBitWidth());
2327 computeKnownBits(V, Known, Depth, Q);
2328 return Mask.isSubsetOf(Known.Zero);
Chris Lattner965c7692008-06-02 01:18:21 +00002329}
2330
Craig Topperbec15b62018-08-22 23:27:50 +00002331// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
2332// Returns the input and lower/upper bounds.
2333static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
2334 const APInt *&CLow, const APInt *&CHigh) {
Craig Topper15f86922018-08-23 17:15:02 +00002335 assert(isa<Operator>(Select) &&
2336 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
Craig Topperdfa176e2018-08-23 17:45:53 +00002337 "Input should be a Select!");
Craig Topperbec15b62018-08-22 23:27:50 +00002338
Simon Pilgrimf62293e2019-09-23 13:15:52 +00002339 const Value *LHS = nullptr, *RHS = nullptr;
Craig Topperbec15b62018-08-22 23:27:50 +00002340 SelectPatternFlavor SPF = matchSelectPattern(Select, LHS, RHS).Flavor;
2341 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
2342 return false;
2343
2344 if (!match(RHS, m_APInt(CLow)))
2345 return false;
2346
Simon Pilgrimf62293e2019-09-23 13:15:52 +00002347 const Value *LHS2 = nullptr, *RHS2 = nullptr;
Craig Topperbec15b62018-08-22 23:27:50 +00002348 SelectPatternFlavor SPF2 = matchSelectPattern(LHS, LHS2, RHS2).Flavor;
2349 if (getInverseMinMaxFlavor(SPF) != SPF2)
2350 return false;
2351
2352 if (!match(RHS2, m_APInt(CHigh)))
2353 return false;
2354
2355 if (SPF == SPF_SMIN)
2356 std::swap(CLow, CHigh);
2357
2358 In = LHS2;
2359 return CLow->sle(*CHigh);
2360}
2361
Sanjay Patela06d9892016-06-22 19:20:59 +00002362/// For vector constants, loop over the elements and find the constant with the
2363/// minimum number of sign bits. Return 0 if the value is not a vector constant
2364/// or if any element was not analyzed; otherwise, return the count for the
2365/// element with the minimum number of sign bits.
Pete Cooper35b00d52016-08-13 01:05:32 +00002366static unsigned computeNumSignBitsVectorConstant(const Value *V,
2367 unsigned TyBits) {
2368 const auto *CV = dyn_cast<Constant>(V);
Sanjay Patela06d9892016-06-22 19:20:59 +00002369 if (!CV || !CV->getType()->isVectorTy())
2370 return 0;
Chris Lattner965c7692008-06-02 01:18:21 +00002371
Sanjay Patela06d9892016-06-22 19:20:59 +00002372 unsigned MinSignBits = TyBits;
2373 unsigned NumElts = CV->getType()->getVectorNumElements();
2374 for (unsigned i = 0; i != NumElts; ++i) {
2375 // If we find a non-ConstantInt, bail out.
2376 auto *Elt = dyn_cast_or_null<ConstantInt>(CV->getAggregateElement(i));
2377 if (!Elt)
2378 return 0;
2379
Craig Topper8e8b6ef2017-10-21 16:35:41 +00002380 MinSignBits = std::min(MinSignBits, Elt->getValue().getNumSignBits());
Sanjay Patela06d9892016-06-22 19:20:59 +00002381 }
2382
2383 return MinSignBits;
2384}
Chris Lattner965c7692008-06-02 01:18:21 +00002385
Sanjoy Das39a684d2017-02-25 20:30:45 +00002386static unsigned ComputeNumSignBitsImpl(const Value *V, unsigned Depth,
2387 const Query &Q);
2388
2389static unsigned ComputeNumSignBits(const Value *V, unsigned Depth,
2390 const Query &Q) {
2391 unsigned Result = ComputeNumSignBitsImpl(V, Depth, Q);
2392 assert(Result > 0 && "At least one sign bit needs to be present!");
2393 return Result;
2394}
2395
Sanjay Patelaee84212014-11-04 16:27:42 +00002396/// Return the number of times the sign bit of the register is replicated into
2397/// the other bits. We know that at least 1 bit is always equal to the sign bit
2398/// (itself), but other cases can give us information. For example, immediately
2399/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
Sanjay Patela06d9892016-06-22 19:20:59 +00002400/// other, so we return 3. For vectors, return the number of sign bits for the
Vedant Kumard3196742018-02-28 19:08:52 +00002401/// vector element with the minimum number of known sign bits.
Sanjoy Das39a684d2017-02-25 20:30:45 +00002402static unsigned ComputeNumSignBitsImpl(const Value *V, unsigned Depth,
2403 const Query &Q) {
Craig Topper7227eba2017-08-21 22:56:12 +00002404 assert(Depth <= MaxDepth && "Limit Search Depth");
Sanjoy Das39a684d2017-02-25 20:30:45 +00002405
2406 // We return the minimum number of sign bits that are guaranteed to be present
2407 // in V, so for undef we have to conservatively return 1. We don't have the
2408 // same behavior for poison though -- that's a FIXME today.
2409
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00002410 Type *ScalarTy = V->getType()->getScalarType();
2411 unsigned TyBits = ScalarTy->isPointerTy() ?
Nicola Zaghenf798eb22019-12-12 10:25:14 +00002412 Q.DL.getIndexTypeSizeInBits(ScalarTy) :
Elena Demikhovsky945b7e52018-02-14 06:58:08 +00002413 Q.DL.getTypeSizeInBits(ScalarTy);
2414
Chris Lattner965c7692008-06-02 01:18:21 +00002415 unsigned Tmp, Tmp2;
2416 unsigned FirstAnswer = 1;
2417
Jay Foada0653a32014-05-14 21:14:37 +00002418 // Note that ConstantInt is handled by the general computeKnownBits case
Chris Lattner2e01a692008-06-02 18:39:07 +00002419 // below.
2420
Matt Arsenaultcb2a7eb2016-12-20 19:06:15 +00002421 if (Depth == MaxDepth)
Chris Lattner965c7692008-06-02 01:18:21 +00002422 return 1; // Limit search depth.
Craig Topper1bef2c82012-12-22 19:15:35 +00002423
Simon Pilgrim2dcee962019-09-26 11:09:08 +00002424 if (auto *U = dyn_cast<Operator>(V)) {
2425 switch (Operator::getOpcode(V)) {
2426 default: break;
2427 case Instruction::SExt:
2428 Tmp = TyBits - U->getOperand(0)->getType()->getScalarSizeInBits();
2429 return ComputeNumSignBits(U->getOperand(0), Depth + 1, Q) + Tmp;
Craig Topper1bef2c82012-12-22 19:15:35 +00002430
Simon Pilgrim2dcee962019-09-26 11:09:08 +00002431 case Instruction::SDiv: {
2432 const APInt *Denominator;
2433 // sdiv X, C -> adds log(C) sign bits.
2434 if (match(U->getOperand(1), m_APInt(Denominator))) {
Nadav Rotem029c5c72015-03-03 21:39:02 +00002435
Simon Pilgrim2dcee962019-09-26 11:09:08 +00002436 // Ignore non-positive denominator.
2437 if (!Denominator->isStrictlyPositive())
2438 break;
Nadav Rotem029c5c72015-03-03 21:39:02 +00002439
Simon Pilgrim2dcee962019-09-26 11:09:08 +00002440 // Calculate the incoming numerator bits.
2441 unsigned NumBits = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
Nadav Rotem029c5c72015-03-03 21:39:02 +00002442
Simon Pilgrim2dcee962019-09-26 11:09:08 +00002443 // Add floor(log(C)) bits to the numerator bits.
2444 return std::min(TyBits, NumBits + Denominator->logBase2());
Chris Lattner965c7692008-06-02 01:18:21 +00002445 }
Sanjay Patelcc9e4012018-10-26 21:05:14 +00002446 break;
Simon Pilgrim2dcee962019-09-26 11:09:08 +00002447 }
2448
2449 case Instruction::SRem: {
2450 const APInt *Denominator;
2451 // srem X, C -> we know that the result is within [-C+1,C) when C is a
2452 // positive constant. This let us put a lower bound on the number of sign
2453 // bits.
2454 if (match(U->getOperand(1), m_APInt(Denominator))) {
2455
2456 // Ignore non-positive denominator.
2457 if (!Denominator->isStrictlyPositive())
2458 break;
2459
2460 // Calculate the incoming numerator bits. SRem by a positive constant
2461 // can't lower the number of sign bits.
2462 unsigned NumrBits = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2463
2464 // Calculate the leading sign bit constraints by examining the
2465 // denominator. Given that the denominator is positive, there are two
2466 // cases:
2467 //
2468 // 1. the numerator is positive. The result range is [0,C) and [0,C) u<
2469 // (1 << ceilLogBase2(C)).
2470 //
2471 // 2. the numerator is negative. Then the result range is (-C,0] and
2472 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
2473 //
2474 // Thus a lower bound on the number of sign bits is `TyBits -
2475 // ceilLogBase2(C)`.
2476
2477 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
2478 return std::max(NumrBits, ResBits);
2479 }
2480 break;
2481 }
2482
2483 case Instruction::AShr: {
2484 Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2485 // ashr X, C -> adds C sign bits. Vectors too.
2486 const APInt *ShAmt;
2487 if (match(U->getOperand(1), m_APInt(ShAmt))) {
2488 if (ShAmt->uge(TyBits))
2489 break; // Bad shift.
2490 unsigned ShAmtLimited = ShAmt->getZExtValue();
2491 Tmp += ShAmtLimited;
2492 if (Tmp > TyBits) Tmp = TyBits;
2493 }
2494 return Tmp;
2495 }
2496 case Instruction::Shl: {
2497 const APInt *ShAmt;
2498 if (match(U->getOperand(1), m_APInt(ShAmt))) {
2499 // shl destroys sign bits.
2500 Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2501 if (ShAmt->uge(TyBits) || // Bad shift.
2502 ShAmt->uge(Tmp)) break; // Shifted all sign bits out.
2503 Tmp2 = ShAmt->getZExtValue();
2504 return Tmp - Tmp2;
2505 }
2506 break;
2507 }
2508 case Instruction::And:
2509 case Instruction::Or:
2510 case Instruction::Xor: // NOT is handled here.
2511 // Logical binary ops preserve the number of sign bits at the worst.
2512 Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2513 if (Tmp != 1) {
2514 Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
2515 FirstAnswer = std::min(Tmp, Tmp2);
2516 // We computed what we know about the sign bits as our first
2517 // answer. Now proceed to the generic code that uses
2518 // computeKnownBits, and pick whichever answer is better.
2519 }
2520 break;
2521
2522 case Instruction::Select: {
2523 // If we have a clamp pattern, we know that the number of sign bits will
2524 // be the minimum of the clamp min/max range.
2525 const Value *X;
2526 const APInt *CLow, *CHigh;
2527 if (isSignedMinMaxClamp(U, X, CLow, CHigh))
2528 return std::min(CLow->getNumSignBits(), CHigh->getNumSignBits());
2529
2530 Tmp = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
2531 if (Tmp == 1) break;
2532 Tmp2 = ComputeNumSignBits(U->getOperand(2), Depth + 1, Q);
2533 return std::min(Tmp, Tmp2);
2534 }
2535
2536 case Instruction::Add:
2537 // Add can have at most one carry bit. Thus we know that the output
2538 // is, at worst, one more bit than the inputs.
2539 Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2540 if (Tmp == 1) break;
2541
2542 // Special case decrementing a value (ADD X, -1):
2543 if (const auto *CRHS = dyn_cast<Constant>(U->getOperand(1)))
2544 if (CRHS->isAllOnesValue()) {
2545 KnownBits Known(TyBits);
2546 computeKnownBits(U->getOperand(0), Known, Depth + 1, Q);
2547
2548 // If the input is known to be 0 or 1, the output is 0/-1, which is
2549 // all sign bits set.
2550 if ((Known.Zero | 1).isAllOnesValue())
2551 return TyBits;
2552
2553 // If we are subtracting one from a positive number, there is no carry
2554 // out of the result.
2555 if (Known.isNonNegative())
2556 return Tmp;
2557 }
2558
2559 Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
2560 if (Tmp2 == 1) break;
2561 return std::min(Tmp, Tmp2) - 1;
2562
2563 case Instruction::Sub:
2564 Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
2565 if (Tmp2 == 1) break;
2566
2567 // Handle NEG.
2568 if (const auto *CLHS = dyn_cast<Constant>(U->getOperand(0)))
2569 if (CLHS->isNullValue()) {
2570 KnownBits Known(TyBits);
2571 computeKnownBits(U->getOperand(1), Known, Depth + 1, Q);
2572 // If the input is known to be 0 or 1, the output is 0/-1, which is
2573 // all sign bits set.
2574 if ((Known.Zero | 1).isAllOnesValue())
2575 return TyBits;
2576
2577 // If the input is known to be positive (the sign bit is known clear),
2578 // the output of the NEG has the same number of sign bits as the
2579 // input.
2580 if (Known.isNonNegative())
2581 return Tmp2;
2582
2583 // Otherwise, we treat this like a SUB.
2584 }
2585
2586 // Sub can have at most one carry bit. Thus we know that the output
2587 // is, at worst, one more bit than the inputs.
2588 Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2589 if (Tmp == 1) break;
2590 return std::min(Tmp, Tmp2) - 1;
2591
2592 case Instruction::Mul: {
2593 // The output of the Mul can be at most twice the valid bits in the
2594 // inputs.
2595 unsigned SignBitsOp0 = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2596 if (SignBitsOp0 == 1) break;
2597 unsigned SignBitsOp1 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
2598 if (SignBitsOp1 == 1) break;
2599 unsigned OutValidBits =
2600 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
2601 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
2602 }
2603
2604 case Instruction::PHI: {
2605 const PHINode *PN = cast<PHINode>(U);
2606 unsigned NumIncomingValues = PN->getNumIncomingValues();
2607 // Don't analyze large in-degree PHIs.
2608 if (NumIncomingValues > 4) break;
2609 // Unreachable blocks may have zero-operand PHI nodes.
2610 if (NumIncomingValues == 0) break;
2611
2612 // Take the minimum of all incoming values. This can't infinitely loop
2613 // because of our depth threshold.
2614 Tmp = ComputeNumSignBits(PN->getIncomingValue(0), Depth + 1, Q);
2615 for (unsigned i = 1, e = NumIncomingValues; i != e; ++i) {
2616 if (Tmp == 1) return Tmp;
2617 Tmp = std::min(
2618 Tmp, ComputeNumSignBits(PN->getIncomingValue(i), Depth + 1, Q));
2619 }
2620 return Tmp;
2621 }
2622
2623 case Instruction::Trunc:
2624 // FIXME: it's tricky to do anything useful for this, but it is an
2625 // important case for targets like X86.
2626 break;
2627
2628 case Instruction::ExtractElement:
2629 // Look through extract element. At the moment we keep this simple and
2630 // skip tracking the specific element. But at least we might find
2631 // information valid for all elements of the vector (for example if vector
2632 // is sign extended, shifted, etc).
2633 return ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
2634
2635 case Instruction::ShuffleVector: {
2636 // TODO: This is copied almost directly from the SelectionDAG version of
2637 // ComputeNumSignBits. It would be better if we could share common
2638 // code. If not, make sure that changes are translated to the DAG.
2639
2640 // Collect the minimum number of sign bits that are shared by every vector
2641 // element referenced by the shuffle.
2642 auto *Shuf = cast<ShuffleVectorInst>(U);
2643 int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
2644 int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
2645 APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2646 for (int i = 0; i != NumMaskElts; ++i) {
2647 int M = Shuf->getMaskValue(i);
2648 assert(M < NumElts * 2 && "Invalid shuffle mask constant");
2649 // For undef elements, we don't know anything about the common state of
2650 // the shuffle result.
2651 if (M == -1)
2652 return 1;
2653 if (M < NumElts)
2654 DemandedLHS.setBit(M % NumElts);
2655 else
2656 DemandedRHS.setBit(M % NumElts);
2657 }
2658 Tmp = std::numeric_limits<unsigned>::max();
2659 if (!!DemandedLHS)
2660 Tmp = ComputeNumSignBits(Shuf->getOperand(0), Depth + 1, Q);
2661 if (!!DemandedRHS) {
2662 Tmp2 = ComputeNumSignBits(Shuf->getOperand(1), Depth + 1, Q);
2663 Tmp = std::min(Tmp, Tmp2);
2664 }
2665 // If we don't know anything, early out and try computeKnownBits
2666 // fall-back.
2667 if (Tmp == 1)
2668 break;
2669 assert(Tmp <= V->getType()->getScalarSizeInBits() &&
2670 "Failed to determine minimum sign bits");
2671 return Tmp;
2672 }
2673 }
Sanjay Patela68096c2018-11-02 15:51:47 +00002674 }
Craig Topper1bef2c82012-12-22 19:15:35 +00002675
Chris Lattner965c7692008-06-02 01:18:21 +00002676 // Finally, if we can prove that the top bits of the result are 0's or 1's,
2677 // use this information.
Sanjay Patela06d9892016-06-22 19:20:59 +00002678
2679 // If we can examine all elements of a vector constant successfully, we're
2680 // done (we can't do any better than that). If not, keep trying.
2681 if (unsigned VecSignBits = computeNumSignBitsVectorConstant(V, TyBits))
2682 return VecSignBits;
2683
Craig Topperb45eabc2017-04-26 16:39:58 +00002684 KnownBits Known(TyBits);
2685 computeKnownBits(V, Known, Depth, Q);
Craig Topper1bef2c82012-12-22 19:15:35 +00002686
Sanjay Patele0536212016-06-23 17:41:59 +00002687 // If we know that the sign bit is either zero or one, determine the number of
2688 // identical bits in the top of the input value.
Craig Topper8df66c62017-05-12 17:20:30 +00002689 return std::max(FirstAnswer, Known.countMinSignBits());
Chris Lattner965c7692008-06-02 01:18:21 +00002690}
Chris Lattnera12a6de2008-06-02 01:29:46 +00002691
Sanjay Patelaee84212014-11-04 16:27:42 +00002692/// This function computes the integer multiple of Base that equals V.
2693/// If successful, it returns true and returns the multiple in
2694/// Multiple. If unsuccessful, it returns false. It looks
Victor Hernandez47444882009-11-10 08:28:35 +00002695/// through SExt instructions only if LookThroughSExt is true.
2696bool llvm::ComputeMultiple(Value *V, unsigned Base, Value *&Multiple,
Dan Gohman6a976bb2009-11-18 00:58:27 +00002697 bool LookThroughSExt, unsigned Depth) {
Dan Gohman6a976bb2009-11-18 00:58:27 +00002698 assert(V && "No Value?");
Victor Hernandez47444882009-11-10 08:28:35 +00002699 assert(Depth <= MaxDepth && "Limit Search Depth");
Duncan Sands9dff9be2010-02-15 16:12:20 +00002700 assert(V->getType()->isIntegerTy() && "Not integer or pointer type!");
Victor Hernandez47444882009-11-10 08:28:35 +00002701
Chris Lattner229907c2011-07-18 04:54:35 +00002702 Type *T = V->getType();
Victor Hernandez47444882009-11-10 08:28:35 +00002703
Dan Gohman6a976bb2009-11-18 00:58:27 +00002704 ConstantInt *CI = dyn_cast<ConstantInt>(V);
Victor Hernandez47444882009-11-10 08:28:35 +00002705
2706 if (Base == 0)
2707 return false;
Craig Topper1bef2c82012-12-22 19:15:35 +00002708
Victor Hernandez47444882009-11-10 08:28:35 +00002709 if (Base == 1) {
2710 Multiple = V;
2711 return true;
2712 }
2713
2714 ConstantExpr *CO = dyn_cast<ConstantExpr>(V);
2715 Constant *BaseVal = ConstantInt::get(T, Base);
2716 if (CO && CO == BaseVal) {
2717 // Multiple is 1.
2718 Multiple = ConstantInt::get(T, 1);
2719 return true;
2720 }
2721
2722 if (CI && CI->getZExtValue() % Base == 0) {
2723 Multiple = ConstantInt::get(T, CI->getZExtValue() / Base);
Craig Topper1bef2c82012-12-22 19:15:35 +00002724 return true;
Victor Hernandez47444882009-11-10 08:28:35 +00002725 }
Craig Topper1bef2c82012-12-22 19:15:35 +00002726
Victor Hernandez47444882009-11-10 08:28:35 +00002727 if (Depth == MaxDepth) return false; // Limit search depth.
Craig Topper1bef2c82012-12-22 19:15:35 +00002728
Victor Hernandez47444882009-11-10 08:28:35 +00002729 Operator *I = dyn_cast<Operator>(V);
2730 if (!I) return false;
2731
2732 switch (I->getOpcode()) {
2733 default: break;
Chris Lattner4f0b47d2009-11-26 01:50:12 +00002734 case Instruction::SExt:
Victor Hernandez47444882009-11-10 08:28:35 +00002735 if (!LookThroughSExt) return false;
2736 // otherwise fall through to ZExt
Galina Kistanova244621f2017-05-31 22:16:24 +00002737 LLVM_FALLTHROUGH;
Chris Lattner4f0b47d2009-11-26 01:50:12 +00002738 case Instruction::ZExt:
Dan Gohman6a976bb2009-11-18 00:58:27 +00002739 return ComputeMultiple(I->getOperand(0), Base, Multiple,
2740 LookThroughSExt, Depth+1);
Victor Hernandez47444882009-11-10 08:28:35 +00002741 case Instruction::Shl:
2742 case Instruction::Mul: {
2743 Value *Op0 = I->getOperand(0);
2744 Value *Op1 = I->getOperand(1);
2745
2746 if (I->getOpcode() == Instruction::Shl) {
2747 ConstantInt *Op1CI = dyn_cast<ConstantInt>(Op1);
2748 if (!Op1CI) return false;
2749 // Turn Op0 << Op1 into Op0 * 2^Op1
2750 APInt Op1Int = Op1CI->getValue();
2751 uint64_t BitToSet = Op1Int.getLimitedValue(Op1Int.getBitWidth() - 1);
Jay Foad15084f02010-11-30 09:02:01 +00002752 APInt API(Op1Int.getBitWidth(), 0);
Jay Foad25a5e4c2010-12-01 08:53:58 +00002753 API.setBit(BitToSet);
Jay Foad15084f02010-11-30 09:02:01 +00002754 Op1 = ConstantInt::get(V->getContext(), API);
Victor Hernandez47444882009-11-10 08:28:35 +00002755 }
2756
Craig Topper9f008862014-04-15 04:59:12 +00002757 Value *Mul0 = nullptr;
Chris Lattner72d283c2010-09-05 17:20:46 +00002758 if (ComputeMultiple(Op0, Base, Mul0, LookThroughSExt, Depth+1)) {
2759 if (Constant *Op1C = dyn_cast<Constant>(Op1))
2760 if (Constant *MulC = dyn_cast<Constant>(Mul0)) {
Craig Topper1bef2c82012-12-22 19:15:35 +00002761 if (Op1C->getType()->getPrimitiveSizeInBits() <
Chris Lattner72d283c2010-09-05 17:20:46 +00002762 MulC->getType()->getPrimitiveSizeInBits())
2763 Op1C = ConstantExpr::getZExt(Op1C, MulC->getType());
Craig Topper1bef2c82012-12-22 19:15:35 +00002764 if (Op1C->getType()->getPrimitiveSizeInBits() >
Chris Lattner72d283c2010-09-05 17:20:46 +00002765 MulC->getType()->getPrimitiveSizeInBits())
2766 MulC = ConstantExpr::getZExt(MulC, Op1C->getType());
Craig Topper1bef2c82012-12-22 19:15:35 +00002767
Chris Lattner72d283c2010-09-05 17:20:46 +00002768 // V == Base * (Mul0 * Op1), so return (Mul0 * Op1)
2769 Multiple = ConstantExpr::getMul(MulC, Op1C);
2770 return true;
2771 }
Victor Hernandez47444882009-11-10 08:28:35 +00002772
2773 if (ConstantInt *Mul0CI = dyn_cast<ConstantInt>(Mul0))
2774 if (Mul0CI->getValue() == 1) {
2775 // V == Base * Op1, so return Op1
2776 Multiple = Op1;
2777 return true;
2778 }
2779 }
2780
Craig Topper9f008862014-04-15 04:59:12 +00002781 Value *Mul1 = nullptr;
Chris Lattner72d283c2010-09-05 17:20:46 +00002782 if (ComputeMultiple(Op1, Base, Mul1, LookThroughSExt, Depth+1)) {
2783 if (Constant *Op0C = dyn_cast<Constant>(Op0))
2784 if (Constant *MulC = dyn_cast<Constant>(Mul1)) {
Craig Topper1bef2c82012-12-22 19:15:35 +00002785 if (Op0C->getType()->getPrimitiveSizeInBits() <
Chris Lattner72d283c2010-09-05 17:20:46 +00002786 MulC->getType()->getPrimitiveSizeInBits())
2787 Op0C = ConstantExpr::getZExt(Op0C, MulC->getType());
Craig Topper1bef2c82012-12-22 19:15:35 +00002788 if (Op0C->getType()->getPrimitiveSizeInBits() >
Chris Lattner72d283c2010-09-05 17:20:46 +00002789 MulC->getType()->getPrimitiveSizeInBits())
2790 MulC = ConstantExpr::getZExt(MulC, Op0C->getType());
Craig Topper1bef2c82012-12-22 19:15:35 +00002791
Chris Lattner72d283c2010-09-05 17:20:46 +00002792 // V == Base * (Mul1 * Op0), so return (Mul1 * Op0)
2793 Multiple = ConstantExpr::getMul(MulC, Op0C);
2794 return true;
2795 }
Victor Hernandez47444882009-11-10 08:28:35 +00002796
2797 if (ConstantInt *Mul1CI = dyn_cast<ConstantInt>(Mul1))
2798 if (Mul1CI->getValue() == 1) {
2799 // V == Base * Op0, so return Op0
2800 Multiple = Op0;
2801 return true;
2802 }
2803 }
Victor Hernandez47444882009-11-10 08:28:35 +00002804 }
2805 }
2806
2807 // We could not determine if V is a multiple of Base.
2808 return false;
2809}
2810
David Majnemerb4b27232016-04-19 19:10:21 +00002811Intrinsic::ID llvm::getIntrinsicForCallSite(ImmutableCallSite ICS,
2812 const TargetLibraryInfo *TLI) {
2813 const Function *F = ICS.getCalledFunction();
2814 if (!F)
2815 return Intrinsic::not_intrinsic;
2816
2817 if (F->isIntrinsic())
2818 return F->getIntrinsicID();
2819
2820 if (!TLI)
2821 return Intrinsic::not_intrinsic;
2822
David L. Jonesd21529f2017-01-23 23:16:46 +00002823 LibFunc Func;
David Majnemerb4b27232016-04-19 19:10:21 +00002824 // We're going to make assumptions on the semantics of the functions, check
2825 // that the target knows that it's available in this environment and it does
2826 // not have local linkage.
Ahmed Bougachad765a822016-04-27 19:04:35 +00002827 if (!F || F->hasLocalLinkage() || !TLI->getLibFunc(*F, Func))
2828 return Intrinsic::not_intrinsic;
2829
2830 if (!ICS.onlyReadsMemory())
David Majnemerb4b27232016-04-19 19:10:21 +00002831 return Intrinsic::not_intrinsic;
2832
2833 // Otherwise check if we have a call to a function that can be turned into a
2834 // vector intrinsic.
2835 switch (Func) {
2836 default:
2837 break;
David L. Jonesd21529f2017-01-23 23:16:46 +00002838 case LibFunc_sin:
2839 case LibFunc_sinf:
2840 case LibFunc_sinl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002841 return Intrinsic::sin;
David L. Jonesd21529f2017-01-23 23:16:46 +00002842 case LibFunc_cos:
2843 case LibFunc_cosf:
2844 case LibFunc_cosl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002845 return Intrinsic::cos;
David L. Jonesd21529f2017-01-23 23:16:46 +00002846 case LibFunc_exp:
2847 case LibFunc_expf:
2848 case LibFunc_expl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002849 return Intrinsic::exp;
David L. Jonesd21529f2017-01-23 23:16:46 +00002850 case LibFunc_exp2:
2851 case LibFunc_exp2f:
2852 case LibFunc_exp2l:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002853 return Intrinsic::exp2;
David L. Jonesd21529f2017-01-23 23:16:46 +00002854 case LibFunc_log:
2855 case LibFunc_logf:
2856 case LibFunc_logl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002857 return Intrinsic::log;
David L. Jonesd21529f2017-01-23 23:16:46 +00002858 case LibFunc_log10:
2859 case LibFunc_log10f:
2860 case LibFunc_log10l:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002861 return Intrinsic::log10;
David L. Jonesd21529f2017-01-23 23:16:46 +00002862 case LibFunc_log2:
2863 case LibFunc_log2f:
2864 case LibFunc_log2l:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002865 return Intrinsic::log2;
David L. Jonesd21529f2017-01-23 23:16:46 +00002866 case LibFunc_fabs:
2867 case LibFunc_fabsf:
2868 case LibFunc_fabsl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002869 return Intrinsic::fabs;
David L. Jonesd21529f2017-01-23 23:16:46 +00002870 case LibFunc_fmin:
2871 case LibFunc_fminf:
2872 case LibFunc_fminl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002873 return Intrinsic::minnum;
David L. Jonesd21529f2017-01-23 23:16:46 +00002874 case LibFunc_fmax:
2875 case LibFunc_fmaxf:
2876 case LibFunc_fmaxl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002877 return Intrinsic::maxnum;
David L. Jonesd21529f2017-01-23 23:16:46 +00002878 case LibFunc_copysign:
2879 case LibFunc_copysignf:
2880 case LibFunc_copysignl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002881 return Intrinsic::copysign;
David L. Jonesd21529f2017-01-23 23:16:46 +00002882 case LibFunc_floor:
2883 case LibFunc_floorf:
2884 case LibFunc_floorl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002885 return Intrinsic::floor;
David L. Jonesd21529f2017-01-23 23:16:46 +00002886 case LibFunc_ceil:
2887 case LibFunc_ceilf:
2888 case LibFunc_ceill:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002889 return Intrinsic::ceil;
David L. Jonesd21529f2017-01-23 23:16:46 +00002890 case LibFunc_trunc:
2891 case LibFunc_truncf:
2892 case LibFunc_truncl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002893 return Intrinsic::trunc;
David L. Jonesd21529f2017-01-23 23:16:46 +00002894 case LibFunc_rint:
2895 case LibFunc_rintf:
2896 case LibFunc_rintl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002897 return Intrinsic::rint;
David L. Jonesd21529f2017-01-23 23:16:46 +00002898 case LibFunc_nearbyint:
2899 case LibFunc_nearbyintf:
2900 case LibFunc_nearbyintl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002901 return Intrinsic::nearbyint;
David L. Jonesd21529f2017-01-23 23:16:46 +00002902 case LibFunc_round:
2903 case LibFunc_roundf:
2904 case LibFunc_roundl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002905 return Intrinsic::round;
David L. Jonesd21529f2017-01-23 23:16:46 +00002906 case LibFunc_pow:
2907 case LibFunc_powf:
2908 case LibFunc_powl:
Ahmed Bougachad765a822016-04-27 19:04:35 +00002909 return Intrinsic::pow;
David L. Jonesd21529f2017-01-23 23:16:46 +00002910 case LibFunc_sqrt:
2911 case LibFunc_sqrtf:
2912 case LibFunc_sqrtl:
Sanjay Patel86d24f12017-11-06 22:40:09 +00002913 return Intrinsic::sqrt;
David Majnemerb4b27232016-04-19 19:10:21 +00002914 }
2915
2916 return Intrinsic::not_intrinsic;
2917}
2918
Sanjay Patelaee84212014-11-04 16:27:42 +00002919/// Return true if we can prove that the specified FP value is never equal to
2920/// -0.0.
Chris Lattnera12a6de2008-06-02 01:29:46 +00002921///
2922/// NOTE: this function will need to be revisited when we support non-default
2923/// rounding modes!
David Majnemer3ee5f342016-04-13 06:55:52 +00002924bool llvm::CannotBeNegativeZero(const Value *V, const TargetLibraryInfo *TLI,
2925 unsigned Depth) {
Sanjay Patel20df88a2017-11-13 17:56:23 +00002926 if (auto *CFP = dyn_cast<ConstantFP>(V))
Chris Lattnera12a6de2008-06-02 01:29:46 +00002927 return !CFP->getValueAPF().isNegZero();
Craig Topper1bef2c82012-12-22 19:15:35 +00002928
Sanjay Patel20df88a2017-11-13 17:56:23 +00002929 // Limit search depth.
Matt Arsenaultcb2a7eb2016-12-20 19:06:15 +00002930 if (Depth == MaxDepth)
Sanjay Patel20df88a2017-11-13 17:56:23 +00002931 return false;
Chris Lattnera12a6de2008-06-02 01:29:46 +00002932
Sanjay Patel20df88a2017-11-13 17:56:23 +00002933 auto *Op = dyn_cast<Operator>(V);
2934 if (!Op)
2935 return false;
Michael Ilseman0f128372012-12-06 00:07:09 +00002936
Sanjay Patel20df88a2017-11-13 17:56:23 +00002937 // Check if the nsz fast-math flag is set.
2938 if (auto *FPO = dyn_cast<FPMathOperator>(Op))
Michael Ilseman0f128372012-12-06 00:07:09 +00002939 if (FPO->hasNoSignedZeros())
2940 return true;
2941
Sanjay Patel9e3d8f42017-11-13 17:40:47 +00002942 // (fadd x, 0.0) is guaranteed to return +0.0, not -0.0.
Sanjay Patel93e64dd2018-03-25 21:16:33 +00002943 if (match(Op, m_FAdd(m_Value(), m_PosZeroFP())))
Sanjay Patel9e3d8f42017-11-13 17:40:47 +00002944 return true;
Craig Topper1bef2c82012-12-22 19:15:35 +00002945
Chris Lattnera12a6de2008-06-02 01:29:46 +00002946 // sitofp and uitofp turn into +0.0 for zero.
Sanjay Patel20df88a2017-11-13 17:56:23 +00002947 if (isa<SIToFPInst>(Op) || isa<UIToFPInst>(Op))
Chris Lattnera12a6de2008-06-02 01:29:46 +00002948 return true;
Craig Topper1bef2c82012-12-22 19:15:35 +00002949
Sanjay Patel20df88a2017-11-13 17:56:23 +00002950 if (auto *Call = dyn_cast<CallInst>(Op)) {
2951 Intrinsic::ID IID = getIntrinsicForCallSite(Call, TLI);
David Majnemer3ee5f342016-04-13 06:55:52 +00002952 switch (IID) {
2953 default:
2954 break;
Chris Lattnera12a6de2008-06-02 01:29:46 +00002955 // sqrt(-0.0) = -0.0, no other negative results are possible.
David Majnemer3ee5f342016-04-13 06:55:52 +00002956 case Intrinsic::sqrt:
Matt Arsenault56b31d82018-08-06 15:16:26 +00002957 case Intrinsic::canonicalize:
Sanjay Patel20df88a2017-11-13 17:56:23 +00002958 return CannotBeNegativeZero(Call->getArgOperand(0), TLI, Depth + 1);
David Majnemer3ee5f342016-04-13 06:55:52 +00002959 // fabs(x) != -0.0
2960 case Intrinsic::fabs:
2961 return true;
Chris Lattnera12a6de2008-06-02 01:29:46 +00002962 }
David Majnemer3ee5f342016-04-13 06:55:52 +00002963 }
Craig Topper1bef2c82012-12-22 19:15:35 +00002964
Chris Lattnera12a6de2008-06-02 01:29:46 +00002965 return false;
2966}
2967
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00002968/// If \p SignBitOnly is true, test for a known 0 sign bit rather than a
2969/// standard ordered compare. e.g. make -0.0 olt 0.0 be true because of the sign
2970/// bit despite comparing equal.
2971static bool cannotBeOrderedLessThanZeroImpl(const Value *V,
2972 const TargetLibraryInfo *TLI,
2973 bool SignBitOnly,
2974 unsigned Depth) {
Justin Lebar322c1272017-01-27 00:58:34 +00002975 // TODO: This function does not do the right thing when SignBitOnly is true
2976 // and we're lowering to a hypothetical IEEE 754-compliant-but-evil platform
2977 // which flips the sign bits of NaNs. See
2978 // https://llvm.org/bugs/show_bug.cgi?id=31702.
2979
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00002980 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(V)) {
2981 return !CFP->getValueAPF().isNegative() ||
2982 (!SignBitOnly && CFP->getValueAPF().isZero());
2983 }
Elena Demikhovsky45f04482015-01-28 08:03:58 +00002984
Craig Topper69c89722018-02-26 22:33:17 +00002985 // Handle vector of constants.
2986 if (auto *CV = dyn_cast<Constant>(V)) {
2987 if (CV->getType()->isVectorTy()) {
2988 unsigned NumElts = CV->getType()->getVectorNumElements();
2989 for (unsigned i = 0; i != NumElts; ++i) {
2990 auto *CFP = dyn_cast_or_null<ConstantFP>(CV->getAggregateElement(i));
2991 if (!CFP)
2992 return false;
2993 if (CFP->getValueAPF().isNegative() &&
2994 (SignBitOnly || !CFP->getValueAPF().isZero()))
2995 return false;
2996 }
2997
2998 // All non-negative ConstantFPs.
2999 return true;
3000 }
3001 }
3002
Matt Arsenaultcb2a7eb2016-12-20 19:06:15 +00003003 if (Depth == MaxDepth)
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003004 return false; // Limit search depth.
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003005
3006 const Operator *I = dyn_cast<Operator>(V);
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003007 if (!I)
3008 return false;
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003009
3010 switch (I->getOpcode()) {
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003011 default:
3012 break;
Fiona Glaserdb7824f2016-01-12 23:37:30 +00003013 // Unsigned integers are always nonnegative.
3014 case Instruction::UIToFP:
3015 return true;
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003016 case Instruction::FMul:
3017 // x*x is always non-negative or a NaN.
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003018 if (I->getOperand(0) == I->getOperand(1) &&
3019 (!SignBitOnly || cast<FPMathOperator>(I)->hasNoNaNs()))
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003020 return true;
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003021
Justin Bognercd1d5aa2016-08-17 20:30:52 +00003022 LLVM_FALLTHROUGH;
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003023 case Instruction::FAdd:
3024 case Instruction::FDiv:
3025 case Instruction::FRem:
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003026 return cannotBeOrderedLessThanZeroImpl(I->getOperand(0), TLI, SignBitOnly,
3027 Depth + 1) &&
3028 cannotBeOrderedLessThanZeroImpl(I->getOperand(1), TLI, SignBitOnly,
3029 Depth + 1);
Fiona Glaserdb7824f2016-01-12 23:37:30 +00003030 case Instruction::Select:
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003031 return cannotBeOrderedLessThanZeroImpl(I->getOperand(1), TLI, SignBitOnly,
3032 Depth + 1) &&
3033 cannotBeOrderedLessThanZeroImpl(I->getOperand(2), TLI, SignBitOnly,
3034 Depth + 1);
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003035 case Instruction::FPExt:
3036 case Instruction::FPTrunc:
3037 // Widening/narrowing never change sign.
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003038 return cannotBeOrderedLessThanZeroImpl(I->getOperand(0), TLI, SignBitOnly,
3039 Depth + 1);
Craig Topper30199102018-02-27 19:53:45 +00003040 case Instruction::ExtractElement:
3041 // Look through extract element. At the moment we keep this simple and skip
3042 // tracking the specific element. But at least we might find information
3043 // valid for all elements of the vector.
3044 return cannotBeOrderedLessThanZeroImpl(I->getOperand(0), TLI, SignBitOnly,
3045 Depth + 1);
David Majnemer3ee5f342016-04-13 06:55:52 +00003046 case Instruction::Call:
Justin Lebar7e3184c2017-01-26 00:10:26 +00003047 const auto *CI = cast<CallInst>(I);
3048 Intrinsic::ID IID = getIntrinsicForCallSite(CI, TLI);
David Majnemer3ee5f342016-04-13 06:55:52 +00003049 switch (IID) {
3050 default:
3051 break;
3052 case Intrinsic::maxnum:
Matt Arsenaultd54b7f02018-08-09 22:40:08 +00003053 return (isKnownNeverNaN(I->getOperand(0), TLI) &&
Sanjay Patelf9a0d592018-08-02 13:46:20 +00003054 cannotBeOrderedLessThanZeroImpl(I->getOperand(0), TLI,
3055 SignBitOnly, Depth + 1)) ||
Matt Arsenaultd54b7f02018-08-09 22:40:08 +00003056 (isKnownNeverNaN(I->getOperand(1), TLI) &&
Sanjay Patelf9a0d592018-08-02 13:46:20 +00003057 cannotBeOrderedLessThanZeroImpl(I->getOperand(1), TLI,
3058 SignBitOnly, Depth + 1));
3059
Thomas Livelyc3392502018-10-19 19:01:26 +00003060 case Intrinsic::maximum:
3061 return cannotBeOrderedLessThanZeroImpl(I->getOperand(0), TLI, SignBitOnly,
3062 Depth + 1) ||
3063 cannotBeOrderedLessThanZeroImpl(I->getOperand(1), TLI, SignBitOnly,
3064 Depth + 1);
David Majnemer3ee5f342016-04-13 06:55:52 +00003065 case Intrinsic::minnum:
Thomas Livelyc3392502018-10-19 19:01:26 +00003066 case Intrinsic::minimum:
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003067 return cannotBeOrderedLessThanZeroImpl(I->getOperand(0), TLI, SignBitOnly,
3068 Depth + 1) &&
3069 cannotBeOrderedLessThanZeroImpl(I->getOperand(1), TLI, SignBitOnly,
3070 Depth + 1);
David Majnemer3ee5f342016-04-13 06:55:52 +00003071 case Intrinsic::exp:
3072 case Intrinsic::exp2:
3073 case Intrinsic::fabs:
David Majnemer3ee5f342016-04-13 06:55:52 +00003074 return true;
Justin Lebar7e3184c2017-01-26 00:10:26 +00003075
3076 case Intrinsic::sqrt:
3077 // sqrt(x) is always >= -0 or NaN. Moreover, sqrt(x) == -0 iff x == -0.
3078 if (!SignBitOnly)
3079 return true;
3080 return CI->hasNoNaNs() && (CI->hasNoSignedZeros() ||
3081 CannotBeNegativeZero(CI->getOperand(0), TLI));
3082
David Majnemer3ee5f342016-04-13 06:55:52 +00003083 case Intrinsic::powi:
Justin Lebar7e3184c2017-01-26 00:10:26 +00003084 if (ConstantInt *Exponent = dyn_cast<ConstantInt>(I->getOperand(1))) {
David Majnemer3ee5f342016-04-13 06:55:52 +00003085 // powi(x,n) is non-negative if n is even.
Justin Lebar7e3184c2017-01-26 00:10:26 +00003086 if (Exponent->getBitWidth() <= 64 && Exponent->getSExtValue() % 2u == 0)
David Majnemer3ee5f342016-04-13 06:55:52 +00003087 return true;
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003088 }
Justin Lebar322c1272017-01-27 00:58:34 +00003089 // TODO: This is not correct. Given that exp is an integer, here are the
3090 // ways that pow can return a negative value:
3091 //
3092 // pow(x, exp) --> negative if exp is odd and x is negative.
3093 // pow(-0, exp) --> -inf if exp is negative odd.
3094 // pow(-0, exp) --> -0 if exp is positive odd.
3095 // pow(-inf, exp) --> -0 if exp is negative odd.
3096 // pow(-inf, exp) --> -inf if exp is positive odd.
3097 //
3098 // Therefore, if !SignBitOnly, we can return true if x >= +0 or x is NaN,
3099 // but we must return false if x == -0. Unfortunately we do not currently
3100 // have a way of expressing this constraint. See details in
3101 // https://llvm.org/bugs/show_bug.cgi?id=31702.
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003102 return cannotBeOrderedLessThanZeroImpl(I->getOperand(0), TLI, SignBitOnly,
3103 Depth + 1);
Justin Lebar322c1272017-01-27 00:58:34 +00003104
David Majnemer3ee5f342016-04-13 06:55:52 +00003105 case Intrinsic::fma:
3106 case Intrinsic::fmuladd:
3107 // x*x+y is non-negative if y is non-negative.
3108 return I->getOperand(0) == I->getOperand(1) &&
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003109 (!SignBitOnly || cast<FPMathOperator>(I)->hasNoNaNs()) &&
3110 cannotBeOrderedLessThanZeroImpl(I->getOperand(2), TLI, SignBitOnly,
3111 Depth + 1);
David Majnemer3ee5f342016-04-13 06:55:52 +00003112 }
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003113 break;
3114 }
Sanjoy Das6082c1a2016-05-07 02:08:15 +00003115 return false;
Elena Demikhovsky45f04482015-01-28 08:03:58 +00003116}
3117
Matt Arsenault1e0edbf2017-01-11 00:33:24 +00003118bool llvm::CannotBeOrderedLessThanZero(const Value *V,
3119 const TargetLibraryInfo *TLI) {
3120 return cannotBeOrderedLessThanZeroImpl(V, TLI, false, 0);
3121}
3122
3123bool llvm::SignBitMustBeZero(const Value *V, const TargetLibraryInfo *TLI) {
3124 return cannotBeOrderedLessThanZeroImpl(V, TLI, true, 0);
3125}
3126
Benjamin Kramercd481132019-11-19 19:12:45 +01003127bool llvm::isKnownNeverInfinity(const Value *V, const TargetLibraryInfo *TLI,
3128 unsigned Depth) {
3129 assert(V->getType()->isFPOrFPVectorTy() && "Querying for Inf on non-FP type");
3130
3131 // If we're told that infinities won't happen, assume they won't.
3132 if (auto *FPMathOp = dyn_cast<FPMathOperator>(V))
3133 if (FPMathOp->hasNoInfs())
3134 return true;
3135
3136 // Handle scalar constants.
3137 if (auto *CFP = dyn_cast<ConstantFP>(V))
3138 return !CFP->isInfinity();
3139
3140 if (Depth == MaxDepth)
3141 return false;
3142
3143 if (auto *Inst = dyn_cast<Instruction>(V)) {
3144 switch (Inst->getOpcode()) {
3145 case Instruction::Select: {
3146 return isKnownNeverInfinity(Inst->getOperand(1), TLI, Depth + 1) &&
3147 isKnownNeverInfinity(Inst->getOperand(2), TLI, Depth + 1);
3148 }
3149 case Instruction::UIToFP:
3150 // If the input type fits into the floating type the result is finite.
3151 return ilogb(APFloat::getLargest(
3152 Inst->getType()->getScalarType()->getFltSemantics())) >=
3153 (int)Inst->getOperand(0)->getType()->getScalarSizeInBits();
3154 default:
3155 break;
3156 }
3157 }
3158
3159 // Bail out for constant expressions, but try to handle vector constants.
3160 if (!V->getType()->isVectorTy() || !isa<Constant>(V))
3161 return false;
3162
3163 // For vectors, verify that each element is not infinity.
3164 unsigned NumElts = V->getType()->getVectorNumElements();
3165 for (unsigned i = 0; i != NumElts; ++i) {
3166 Constant *Elt = cast<Constant>(V)->getAggregateElement(i);
3167 if (!Elt)
3168 return false;
3169 if (isa<UndefValue>(Elt))
3170 continue;
3171 auto *CElt = dyn_cast<ConstantFP>(Elt);
3172 if (!CElt || CElt->isInfinity())
3173 return false;
3174 }
3175 // All elements were confirmed non-infinity or undefined.
3176 return true;
3177}
3178
Matt Arsenaultd54b7f02018-08-09 22:40:08 +00003179bool llvm::isKnownNeverNaN(const Value *V, const TargetLibraryInfo *TLI,
3180 unsigned Depth) {
Sanjay Patel6840c5f2017-09-05 23:13:13 +00003181 assert(V->getType()->isFPOrFPVectorTy() && "Querying for NaN on non-FP type");
3182
3183 // If we're told that NaNs won't happen, assume they won't.
3184 if (auto *FPMathOp = dyn_cast<FPMathOperator>(V))
3185 if (FPMathOp->hasNoNaNs())
3186 return true;
3187
Sanjay Patel6840c5f2017-09-05 23:13:13 +00003188 // Handle scalar constants.
3189 if (auto *CFP = dyn_cast<ConstantFP>(V))
3190 return !CFP->isNaN();
3191
Matt Arsenaultd54b7f02018-08-09 22:40:08 +00003192 if (Depth == MaxDepth)
3193 return false;
3194
Matt Arsenault450fcc72018-08-20 16:51:00 +00003195 if (auto *Inst = dyn_cast<Instruction>(V)) {
3196 switch (Inst->getOpcode()) {
3197 case Instruction::FAdd:
Matt Arsenault450fcc72018-08-20 16:51:00 +00003198 case Instruction::FSub:
Benjamin Kramercd481132019-11-19 19:12:45 +01003199 // Adding positive and negative infinity produces NaN.
3200 return isKnownNeverNaN(Inst->getOperand(0), TLI, Depth + 1) &&
3201 isKnownNeverNaN(Inst->getOperand(1), TLI, Depth + 1) &&
3202 (isKnownNeverInfinity(Inst->getOperand(0), TLI, Depth + 1) ||
3203 isKnownNeverInfinity(Inst->getOperand(1), TLI, Depth + 1));
3204
3205 case Instruction::FMul:
3206 // Zero multiplied with infinity produces NaN.
3207 // FIXME: If neither side can be zero fmul never produces NaN.
3208 return isKnownNeverNaN(Inst->getOperand(0), TLI, Depth + 1) &&
3209 isKnownNeverInfinity(Inst->getOperand(0), TLI, Depth + 1) &&
3210 isKnownNeverNaN(Inst->getOperand(1), TLI, Depth + 1) &&
3211 isKnownNeverInfinity(Inst->getOperand(1), TLI, Depth + 1);
3212
Matt Arsenault450fcc72018-08-20 16:51:00 +00003213 case Instruction::FDiv:
Benjamin Kramercd481132019-11-19 19:12:45 +01003214 case Instruction::FRem:
3215 // FIXME: Only 0/0, Inf/Inf, Inf REM x and x REM 0 produce NaN.
Matt Arsenault450fcc72018-08-20 16:51:00 +00003216 return false;
Benjamin Kramercd481132019-11-19 19:12:45 +01003217
Matt Arsenault450fcc72018-08-20 16:51:00 +00003218 case Instruction::Select: {
3219 return isKnownNeverNaN(Inst->getOperand(1), TLI, Depth + 1) &&
3220 isKnownNeverNaN(Inst->getOperand(2), TLI, Depth + 1);
3221 }
3222 case Instruction::SIToFP:
3223 case Instruction::UIToFP:
3224 return true;
3225 case Instruction::FPTrunc:
3226 case Instruction::FPExt:
3227 return isKnownNeverNaN(Inst->getOperand(0), TLI, Depth + 1);
3228 default:
3229 break;
3230 }
3231 }
3232
Matt Arsenaultd54b7f02018-08-09 22:40:08 +00003233 if (const auto *II = dyn_cast<IntrinsicInst>(V)) {
3234 switch (II->getIntrinsicID()) {
3235 case Intrinsic::canonicalize:
3236 case Intrinsic::fabs:
3237 case Intrinsic::copysign:
Matt Arsenault450fcc72018-08-20 16:51:00 +00003238 case Intrinsic::exp:
3239 case Intrinsic::exp2:
3240 case Intrinsic::floor:
3241 case Intrinsic::ceil:
3242 case Intrinsic::trunc:
3243 case Intrinsic::rint:
3244 case Intrinsic::nearbyint:
3245 case Intrinsic::round:
Matt Arsenaultd54b7f02018-08-09 22:40:08 +00003246 return isKnownNeverNaN(II->getArgOperand(0), TLI, Depth + 1);
3247 case Intrinsic::sqrt:
3248 return isKnownNeverNaN(II->getArgOperand(0), TLI, Depth + 1) &&
3249 CannotBeOrderedLessThanZero(II->getArgOperand(0), TLI);
Sanjay Patele088d032019-05-07 22:58:31 +00003250 case Intrinsic::minnum:
3251 case Intrinsic::maxnum:
3252 // If either operand is not NaN, the result is not NaN.
3253 return isKnownNeverNaN(II->getArgOperand(0), TLI, Depth + 1) ||
3254 isKnownNeverNaN(II->getArgOperand(1), TLI, Depth + 1);
Matt Arsenaultd54b7f02018-08-09 22:40:08 +00003255 default:
3256 return false;
3257 }
3258 }
3259
Sanjay Patel6840c5f2017-09-05 23:13:13 +00003260 // Bail out for constant expressions, but try to handle vector constants.
3261 if (!V->getType()->isVectorTy() || !isa<Constant>(V))
3262 return false;
3263
3264 // For vectors, verify that each element is not NaN.
3265 unsigned NumElts = V->getType()->getVectorNumElements();
3266 for (unsigned i = 0; i != NumElts; ++i) {
3267 Constant *Elt = cast<Constant>(V)->getAggregateElement(i);
3268 if (!Elt)
3269 return false;
3270 if (isa<UndefValue>(Elt))
3271 continue;
3272 auto *CElt = dyn_cast<ConstantFP>(Elt);
3273 if (!CElt || CElt->isNaN())
3274 return false;
3275 }
3276 // All elements were confirmed not-NaN or undefined.
3277 return true;
3278}
3279
Vitaly Bukad03bd1d2019-07-10 22:53:52 +00003280Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
JF Bastien73d8e4e2018-09-21 05:17:42 +00003281
Chris Lattner9cb10352010-12-26 20:15:01 +00003282 // All byte-wide stores are splatable, even of arbitrary variables.
JF Bastien73d8e4e2018-09-21 05:17:42 +00003283 if (V->getType()->isIntegerTy(8))
3284 return V;
3285
3286 LLVMContext &Ctx = V->getContext();
3287
3288 // Undef don't care.
3289 auto *UndefInt8 = UndefValue::get(Type::getInt8Ty(Ctx));
3290 if (isa<UndefValue>(V))
3291 return UndefInt8;
3292
Vitaly Buka52096ee2019-07-12 02:23:07 +00003293 const uint64_t Size = DL.getTypeStoreSize(V->getType());
3294 if (!Size)
3295 return UndefInt8;
3296
JF Bastien73d8e4e2018-09-21 05:17:42 +00003297 Constant *C = dyn_cast<Constant>(V);
3298 if (!C) {
3299 // Conceptually, we could handle things like:
3300 // %a = zext i8 %X to i16
3301 // %b = shl i16 %a, 8
3302 // %c = or i16 %a, %b
3303 // but until there is an example that actually needs this, it doesn't seem
3304 // worth worrying about.
3305 return nullptr;
3306 }
Chris Lattneracf6b072011-02-19 19:35:49 +00003307
3308 // Handle 'null' ConstantArrayZero etc.
JF Bastien73d8e4e2018-09-21 05:17:42 +00003309 if (C->isNullValue())
3310 return Constant::getNullValue(Type::getInt8Ty(Ctx));
Craig Topper1bef2c82012-12-22 19:15:35 +00003311
JF Bastien73d8e4e2018-09-21 05:17:42 +00003312 // Constant floating-point values can be handled as integer values if the
Craig Topper1bef2c82012-12-22 19:15:35 +00003313 // corresponding integer value is "byteable". An important case is 0.0.
JF Bastien73d8e4e2018-09-21 05:17:42 +00003314 if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
3315 Type *Ty = nullptr;
3316 if (CFP->getType()->isHalfTy())
3317 Ty = Type::getInt16Ty(Ctx);
3318 else if (CFP->getType()->isFloatTy())
3319 Ty = Type::getInt32Ty(Ctx);
3320 else if (CFP->getType()->isDoubleTy())
3321 Ty = Type::getInt64Ty(Ctx);
Chris Lattner9cb10352010-12-26 20:15:01 +00003322 // Don't handle long double formats, which have strange constraints.
Vitaly Bukad03bd1d2019-07-10 22:53:52 +00003323 return Ty ? isBytewiseValue(ConstantExpr::getBitCast(CFP, Ty), DL)
3324 : nullptr;
Chris Lattner9cb10352010-12-26 20:15:01 +00003325 }
Craig Topper1bef2c82012-12-22 19:15:35 +00003326
Benjamin Kramer17d90152015-02-07 19:29:02 +00003327 // We can handle constant integers that are multiple of 8 bits.
JF Bastien73d8e4e2018-09-21 05:17:42 +00003328 if (ConstantInt *CI = dyn_cast<ConstantInt>(C)) {
Benjamin Kramer17d90152015-02-07 19:29:02 +00003329 if (CI->getBitWidth() % 8 == 0) {
3330 assert(CI->getBitWidth() > 8 && "8 bits should be handled above!");
Benjamin Kramerb4b51502015-03-25 16:49:59 +00003331 if (!CI->getValue().isSplat(8))
Benjamin Kramer17d90152015-02-07 19:29:02 +00003332 return nullptr;
JF Bastien73d8e4e2018-09-21 05:17:42 +00003333 return ConstantInt::get(Ctx, CI->getValue().trunc(8));
Chris Lattner9cb10352010-12-26 20:15:01 +00003334 }
3335 }
Craig Topper1bef2c82012-12-22 19:15:35 +00003336
Vitaly Bukac559e632019-07-12 01:42:03 +00003337 if (auto *CE = dyn_cast<ConstantExpr>(C)) {
3338 if (CE->getOpcode() == Instruction::IntToPtr) {
3339 auto PS = DL.getPointerSizeInBits(
3340 cast<PointerType>(CE->getType())->getAddressSpace());
3341 return isBytewiseValue(
3342 ConstantExpr::getIntegerCast(CE->getOperand(0),
3343 Type::getIntNTy(Ctx, PS), false),
3344 DL);
3345 }
3346 }
3347
JF Bastien73d8e4e2018-09-21 05:17:42 +00003348 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
3349 if (LHS == RHS)
3350 return LHS;
3351 if (!LHS || !RHS)
Craig Topper9f008862014-04-15 04:59:12 +00003352 return nullptr;
JF Bastien73d8e4e2018-09-21 05:17:42 +00003353 if (LHS == UndefInt8)
3354 return RHS;
3355 if (RHS == UndefInt8)
3356 return LHS;
3357 return nullptr;
3358 };
Craig Topper1bef2c82012-12-22 19:15:35 +00003359
JF Bastien73d8e4e2018-09-21 05:17:42 +00003360 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(C)) {
3361 Value *Val = UndefInt8;
3362 for (unsigned I = 0, E = CA->getNumElements(); I != E; ++I)
Vitaly Bukad03bd1d2019-07-10 22:53:52 +00003363 if (!(Val = Merge(Val, isBytewiseValue(CA->getElementAsConstant(I), DL))))
Craig Topper9f008862014-04-15 04:59:12 +00003364 return nullptr;
Chris Lattner9cb10352010-12-26 20:15:01 +00003365 return Val;
3366 }
Chad Rosier8abf65a2011-12-06 00:19:08 +00003367
Vitaly Bukab1bff762019-07-12 22:37:55 +00003368 if (isa<ConstantAggregate>(C)) {
JF Bastien73d8e4e2018-09-21 05:17:42 +00003369 Value *Val = UndefInt8;
3370 for (unsigned I = 0, E = C->getNumOperands(); I != E; ++I)
Vitaly Bukad03bd1d2019-07-10 22:53:52 +00003371 if (!(Val = Merge(Val, isBytewiseValue(C->getOperand(I), DL))))
JF Bastien73d8e4e2018-09-21 05:17:42 +00003372 return nullptr;
3373 return Val;
3374 }
3375
3376 // Don't try to handle the handful of other constants.
Craig Topper9f008862014-04-15 04:59:12 +00003377 return nullptr;
Chris Lattner9cb10352010-12-26 20:15:01 +00003378}
3379
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003380// This is the recursive version of BuildSubAggregate. It takes a few different
3381// arguments. Idxs is the index within the nested struct From that we are
3382// looking at now (which is of type IndexedType). IdxSkip is the number of
3383// indices from Idxs that should be left out when inserting into the resulting
3384// struct. To is the result struct built so far, new insertvalue instructions
3385// build on that.
Chris Lattner229907c2011-07-18 04:54:35 +00003386static Value *BuildSubAggregate(Value *From, Value* To, Type *IndexedType,
Craig Topper2cd5ff82013-07-11 16:22:38 +00003387 SmallVectorImpl<unsigned> &Idxs,
Dan Gohmana6d0afc2009-08-07 01:32:21 +00003388 unsigned IdxSkip,
Dan Gohmana6d0afc2009-08-07 01:32:21 +00003389 Instruction *InsertBefore) {
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003390 StructType *STy = dyn_cast<StructType>(IndexedType);
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003391 if (STy) {
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003392 // Save the original To argument so we can modify it
3393 Value *OrigTo = To;
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003394 // General case, the type indexed by Idxs is a struct
3395 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3396 // Process each struct element recursively
3397 Idxs.push_back(i);
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003398 Value *PrevTo = To;
Matthijs Kooijman5cb38772008-06-16 12:57:37 +00003399 To = BuildSubAggregate(From, To, STy->getElementType(i), Idxs, IdxSkip,
Nick Lewycky39dbfd32009-11-23 03:29:18 +00003400 InsertBefore);
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003401 Idxs.pop_back();
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003402 if (!To) {
3403 // Couldn't find any inserted value for this index? Cleanup
3404 while (PrevTo != OrigTo) {
3405 InsertValueInst* Del = cast<InsertValueInst>(PrevTo);
3406 PrevTo = Del->getAggregateOperand();
3407 Del->eraseFromParent();
3408 }
3409 // Stop processing elements
3410 break;
3411 }
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003412 }
Chris Lattner0ab5e2c2011-04-15 05:18:47 +00003413 // If we successfully found a value for each of our subaggregates
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003414 if (To)
3415 return To;
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003416 }
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003417 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
3418 // the struct's elements had a value that was inserted directly. In the latter
3419 // case, perhaps we can't determine each of the subelements individually, but
3420 // we might be able to find the complete struct somewhere.
Craig Topper1bef2c82012-12-22 19:15:35 +00003421
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003422 // Find the value that is at that particular spot
Jay Foad57aa6362011-07-13 10:26:04 +00003423 Value *V = FindInsertedValue(From, Idxs);
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003424
3425 if (!V)
Craig Topper9f008862014-04-15 04:59:12 +00003426 return nullptr;
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003427
Vedant Kumard3196742018-02-28 19:08:52 +00003428 // Insert the value in the new (sub) aggregate
Eugene Zelenko75075ef2017-09-01 21:37:29 +00003429 return InsertValueInst::Create(To, V, makeArrayRef(Idxs).slice(IdxSkip),
3430 "tmp", InsertBefore);
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003431}
3432
3433// This helper takes a nested struct and extracts a part of it (which is again a
3434// struct) into a new value. For example, given the struct:
3435// { a, { b, { c, d }, e } }
3436// and the indices "1, 1" this returns
3437// { c, d }.
3438//
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003439// It does this by inserting an insertvalue for each element in the resulting
3440// struct, as opposed to just inserting a single struct. This will only work if
3441// each of the elements of the substruct are known (ie, inserted into From by an
3442// insertvalue instruction somewhere).
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003443//
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003444// All inserted insertvalue instructions are inserted before InsertBefore
Jay Foad57aa6362011-07-13 10:26:04 +00003445static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
Dan Gohmana6d0afc2009-08-07 01:32:21 +00003446 Instruction *InsertBefore) {
Matthijs Kooijman69801d42008-06-16 13:28:31 +00003447 assert(InsertBefore && "Must have someplace to insert!");
Chris Lattner229907c2011-07-18 04:54:35 +00003448 Type *IndexedType = ExtractValueInst::getIndexedType(From->getType(),
Jay Foad57aa6362011-07-13 10:26:04 +00003449 idx_range);
Owen Andersonb292b8c2009-07-30 23:03:37 +00003450 Value *To = UndefValue::get(IndexedType);
Jay Foad57aa6362011-07-13 10:26:04 +00003451 SmallVector<unsigned, 10> Idxs(idx_range.begin(), idx_range.end());
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003452 unsigned IdxSkip = Idxs.size();
3453
Nick Lewycky39dbfd32009-11-23 03:29:18 +00003454 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003455}
3456
Vedant Kumard3196742018-02-28 19:08:52 +00003457/// Given an aggregate and a sequence of indices, see if the scalar value
3458/// indexed is already around as a register, for example if it was inserted
3459/// directly into the aggregate.
Matthijs Kooijmanfa4d0b82008-06-16 14:13:46 +00003460///
3461/// If InsertBefore is not null, this function will duplicate (modified)
3462/// insertvalues when a part of a nested struct is extracted.
Jay Foad57aa6362011-07-13 10:26:04 +00003463Value *llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
3464 Instruction *InsertBefore) {
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003465 // Nothing to index? Just return V then (this is useful at the end of our
Chris Lattnerf7eb5432012-01-24 07:54:10 +00003466 // recursion).
Jay Foad57aa6362011-07-13 10:26:04 +00003467 if (idx_range.empty())
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003468 return V;
Chris Lattnerf7eb5432012-01-24 07:54:10 +00003469 // We have indices, so V should have an indexable type.
3470 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
3471 "Not looking at a struct or array?");
3472 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
3473 "Invalid indices for type?");
Owen Andersonf1f17432009-07-06 22:37:39 +00003474
Chris Lattner67058832012-01-25 06:48:06 +00003475 if (Constant *C = dyn_cast<Constant>(V)) {
3476 C = C->getAggregateElement(idx_range[0]);
Craig Topper9f008862014-04-15 04:59:12 +00003477 if (!C) return nullptr;
Chris Lattner67058832012-01-25 06:48:06 +00003478 return FindInsertedValue(C, idx_range.slice(1), InsertBefore);
3479 }
Craig Topper1bef2c82012-12-22 19:15:35 +00003480
Chris Lattnerf7eb5432012-01-24 07:54:10 +00003481 if (InsertValueInst *I = dyn_cast<InsertValueInst>(V)) {
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003482 // Loop the indices for the insertvalue instruction in parallel with the
3483 // requested indices
Jay Foad57aa6362011-07-13 10:26:04 +00003484 const unsigned *req_idx = idx_range.begin();
Matthijs Kooijman5cb38772008-06-16 12:57:37 +00003485 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
3486 i != e; ++i, ++req_idx) {
Jay Foad57aa6362011-07-13 10:26:04 +00003487 if (req_idx == idx_range.end()) {
Chris Lattnerf7eb5432012-01-24 07:54:10 +00003488 // We can't handle this without inserting insertvalues
3489 if (!InsertBefore)
Craig Topper9f008862014-04-15 04:59:12 +00003490 return nullptr;
Chris Lattnerf7eb5432012-01-24 07:54:10 +00003491
3492 // The requested index identifies a part of a nested aggregate. Handle
3493 // this specially. For example,
3494 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
3495 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
3496 // %C = extractvalue {i32, { i32, i32 } } %B, 1
3497 // This can be changed into
3498 // %A = insertvalue {i32, i32 } undef, i32 10, 0
3499 // %C = insertvalue {i32, i32 } %A, i32 11, 1
3500 // which allows the unused 0,0 element from the nested struct to be
3501 // removed.
3502 return BuildSubAggregate(V, makeArrayRef(idx_range.begin(), req_idx),
3503 InsertBefore);
Duncan Sandsdb356ee2008-06-19 08:47:31 +00003504 }
Craig Topper1bef2c82012-12-22 19:15:35 +00003505
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003506 // This insert value inserts something else than what we are looking for.
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00003507 // See if the (aggregate) value inserted into has the value we are
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003508 // looking for, then.
3509 if (*req_idx != *i)
Jay Foad57aa6362011-07-13 10:26:04 +00003510 return FindInsertedValue(I->getAggregateOperand(), idx_range,
Nick Lewycky39dbfd32009-11-23 03:29:18 +00003511 InsertBefore);
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003512 }
3513 // If we end up here, the indices of the insertvalue match with those
3514 // requested (though possibly only partially). Now we recursively look at
3515 // the inserted value, passing any remaining indices.
Jay Foad57aa6362011-07-13 10:26:04 +00003516 return FindInsertedValue(I->getInsertedValueOperand(),
Frits van Bommel717d7ed2011-07-18 12:00:32 +00003517 makeArrayRef(req_idx, idx_range.end()),
Nick Lewycky39dbfd32009-11-23 03:29:18 +00003518 InsertBefore);
Chris Lattnerf7eb5432012-01-24 07:54:10 +00003519 }
Craig Topper1bef2c82012-12-22 19:15:35 +00003520
Chris Lattnerf7eb5432012-01-24 07:54:10 +00003521 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(V)) {
Benjamin Kramerdf005cb2015-08-08 18:27:36 +00003522 // If we're extracting a value from an aggregate that was extracted from
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003523 // something else, we can extract from that something else directly instead.
3524 // However, we will need to chain I's indices with the requested indices.
Craig Topper1bef2c82012-12-22 19:15:35 +00003525
3526 // Calculate the number of indices required
Jay Foad57aa6362011-07-13 10:26:04 +00003527 unsigned size = I->getNumIndices() + idx_range.size();
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003528 // Allocate some space to put the new indices in
Matthijs Kooijman8369c672008-06-17 08:24:37 +00003529 SmallVector<unsigned, 5> Idxs;
3530 Idxs.reserve(size);
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003531 // Add indices from the extract value instruction
Jay Foad57aa6362011-07-13 10:26:04 +00003532 Idxs.append(I->idx_begin(), I->idx_end());
Craig Topper1bef2c82012-12-22 19:15:35 +00003533
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003534 // Add requested indices
Jay Foad57aa6362011-07-13 10:26:04 +00003535 Idxs.append(idx_range.begin(), idx_range.end());
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003536
Craig Topper1bef2c82012-12-22 19:15:35 +00003537 assert(Idxs.size() == size
Matthijs Kooijman5cb38772008-06-16 12:57:37 +00003538 && "Number of indices added not correct?");
Craig Topper1bef2c82012-12-22 19:15:35 +00003539
Jay Foad57aa6362011-07-13 10:26:04 +00003540 return FindInsertedValue(I->getAggregateOperand(), Idxs, InsertBefore);
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003541 }
3542 // Otherwise, we don't know (such as, extracting from a function return value
3543 // or load instruction)
Craig Topper9f008862014-04-15 04:59:12 +00003544 return nullptr;
Matthijs Kooijmane92e18b2008-06-16 12:48:21 +00003545}
Evan Chengda3db112008-06-30 07:31:25 +00003546
Matthias Braun50ec0b52017-05-19 22:37:09 +00003547bool llvm::isGEPBasedOnPointerToString(const GEPOperator *GEP,
3548 unsigned CharSize) {
David L Kreitzer752c1442016-04-13 14:31:06 +00003549 // Make sure the GEP has exactly three arguments.
3550 if (GEP->getNumOperands() != 3)
3551 return false;
3552
Matthias Braun50ec0b52017-05-19 22:37:09 +00003553 // Make sure the index-ee is a pointer to array of \p CharSize integers.
3554 // CharSize.
David L Kreitzer752c1442016-04-13 14:31:06 +00003555 ArrayType *AT = dyn_cast<ArrayType>(GEP->getSourceElementType());
Matthias Braun50ec0b52017-05-19 22:37:09 +00003556 if (!AT || !AT->getElementType()->isIntegerTy(CharSize))
David L Kreitzer752c1442016-04-13 14:31:06 +00003557 return false;
3558
3559 // Check to make sure that the first operand of the GEP is an integer and
3560 // has value 0 so that we are sure we're indexing into the initializer.
3561 const ConstantInt *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));
3562 if (!FirstIdx || !FirstIdx->isZero())
3563 return false;
3564
3565 return true;
Sanjoy Das6082c1a2016-05-07 02:08:15 +00003566}
Chris Lattnere28618d2010-11-30 22:25:26 +00003567
Matthias Braun50ec0b52017-05-19 22:37:09 +00003568bool llvm::getConstantDataArrayInfo(const Value *V,
3569 ConstantDataArraySlice &Slice,
3570 unsigned ElementSize, uint64_t Offset) {
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003571 assert(V);
Evan Chengda3db112008-06-30 07:31:25 +00003572
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003573 // Look through bitcast instructions and geps.
3574 V = V->stripPointerCasts();
Craig Topper1bef2c82012-12-22 19:15:35 +00003575
Benjamin Kramer0248a3e2015-03-21 15:36:06 +00003576 // If the value is a GEP instruction or constant expression, treat it as an
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003577 // offset.
3578 if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
David L Kreitzer752c1442016-04-13 14:31:06 +00003579 // The GEP operator should be based on a pointer to string constant, and is
3580 // indexing into the string constant.
Matthias Braun50ec0b52017-05-19 22:37:09 +00003581 if (!isGEPBasedOnPointerToString(GEP, ElementSize))
Bill Wendlingfa54bc22009-03-13 04:39:26 +00003582 return false;
Craig Topper1bef2c82012-12-22 19:15:35 +00003583
Evan Chengda3db112008-06-30 07:31:25 +00003584 // If the second index isn't a ConstantInt, then this is a variable index
3585 // into the array. If this occurs, we can't say anything meaningful about
3586 // the string.
3587 uint64_t StartIdx = 0;
Dan Gohman0b4df042010-04-14 22:20:45 +00003588 if (const ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(2)))
Evan Chengda3db112008-06-30 07:31:25 +00003589 StartIdx = CI->getZExtValue();
Bill Wendlingfa54bc22009-03-13 04:39:26 +00003590 else
3591 return false;
Matthias Braun50ec0b52017-05-19 22:37:09 +00003592 return getConstantDataArrayInfo(GEP->getOperand(0), Slice, ElementSize,
3593 StartIdx + Offset);
Evan Chengda3db112008-06-30 07:31:25 +00003594 }
Nick Lewycky46209882011-10-20 00:34:35 +00003595
Evan Chengda3db112008-06-30 07:31:25 +00003596 // The GEP instruction, constant or instruction, must reference a global
3597 // variable that is a constant and is initialized. The referenced constant
3598 // initializer is the array that we'll use for optimization.
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003599 const GlobalVariable *GV = dyn_cast<GlobalVariable>(V);
Dan Gohman5d5bc6d2009-08-19 18:20:44 +00003600 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
Bill Wendlingfa54bc22009-03-13 04:39:26 +00003601 return false;
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003602
Matthias Braun50ec0b52017-05-19 22:37:09 +00003603 const ConstantDataArray *Array;
3604 ArrayType *ArrayTy;
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003605 if (GV->getInitializer()->isNullValue()) {
Matthias Braun50ec0b52017-05-19 22:37:09 +00003606 Type *GVTy = GV->getValueType();
3607 if ( (ArrayTy = dyn_cast<ArrayType>(GVTy)) ) {
Sanjay Patel2ad88f82017-06-12 22:34:37 +00003608 // A zeroinitializer for the array; there is no ConstantDataArray.
Matthias Braun50ec0b52017-05-19 22:37:09 +00003609 Array = nullptr;
3610 } else {
3611 const DataLayout &DL = GV->getParent()->getDataLayout();
3612 uint64_t SizeInBytes = DL.getTypeStoreSize(GVTy);
3613 uint64_t Length = SizeInBytes / (ElementSize / 8);
3614 if (Length <= Offset)
3615 return false;
Craig Topper1bef2c82012-12-22 19:15:35 +00003616
Matthias Braun50ec0b52017-05-19 22:37:09 +00003617 Slice.Array = nullptr;
3618 Slice.Offset = 0;
3619 Slice.Length = Length - Offset;
3620 return true;
3621 }
3622 } else {
3623 // This must be a ConstantDataArray.
3624 Array = dyn_cast<ConstantDataArray>(GV->getInitializer());
3625 if (!Array)
3626 return false;
3627 ArrayTy = Array->getType();
3628 }
3629 if (!ArrayTy->getElementType()->isIntegerTy(ElementSize))
Bill Wendlingfa54bc22009-03-13 04:39:26 +00003630 return false;
Craig Topper1bef2c82012-12-22 19:15:35 +00003631
Matthias Braun50ec0b52017-05-19 22:37:09 +00003632 uint64_t NumElts = ArrayTy->getArrayNumElements();
Bill Wendlingfa54bc22009-03-13 04:39:26 +00003633 if (Offset > NumElts)
3634 return false;
Craig Topper1bef2c82012-12-22 19:15:35 +00003635
Matthias Braun50ec0b52017-05-19 22:37:09 +00003636 Slice.Array = Array;
3637 Slice.Offset = Offset;
3638 Slice.Length = NumElts - Offset;
3639 return true;
3640}
3641
3642/// This function computes the length of a null-terminated C string pointed to
3643/// by V. If successful, it returns true and returns the string in Str.
3644/// If unsuccessful, it returns false.
3645bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
3646 uint64_t Offset, bool TrimAtNul) {
3647 ConstantDataArraySlice Slice;
3648 if (!getConstantDataArrayInfo(V, Slice, 8, Offset))
3649 return false;
3650
3651 if (Slice.Array == nullptr) {
3652 if (TrimAtNul) {
3653 Str = StringRef();
3654 return true;
3655 }
3656 if (Slice.Length == 1) {
3657 Str = StringRef("", 1);
3658 return true;
3659 }
Sanjay Patelfef83e82017-06-09 14:21:18 +00003660 // We cannot instantiate a StringRef as we do not have an appropriate string
Matthias Braun50ec0b52017-05-19 22:37:09 +00003661 // of 0s at hand.
3662 return false;
3663 }
3664
3665 // Start out with the entire array in the StringRef.
3666 Str = Slice.Array->getAsString();
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003667 // Skip over 'offset' bytes.
Matthias Braun50ec0b52017-05-19 22:37:09 +00003668 Str = Str.substr(Slice.Offset);
Craig Topper1bef2c82012-12-22 19:15:35 +00003669
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003670 if (TrimAtNul) {
3671 // Trim off the \0 and anything after it. If the array is not nul
3672 // terminated, we just return the whole end of string. The client may know
3673 // some other way that the string is length-bound.
3674 Str = Str.substr(0, Str.find('\0'));
3675 }
Bill Wendlingfa54bc22009-03-13 04:39:26 +00003676 return true;
Evan Chengda3db112008-06-30 07:31:25 +00003677}
Eric Christopher4899cbc2010-03-05 06:58:57 +00003678
3679// These next two are very similar to the above, but also look through PHI
3680// nodes.
3681// TODO: See if we can integrate these two together.
3682
Sanjay Patelaee84212014-11-04 16:27:42 +00003683/// If we can compute the length of the string pointed to by
Eric Christopher4899cbc2010-03-05 06:58:57 +00003684/// the specified pointer, return 'len+1'. If we can't, return 0.
Pete Cooper35b00d52016-08-13 01:05:32 +00003685static uint64_t GetStringLengthH(const Value *V,
Matthias Braun50ec0b52017-05-19 22:37:09 +00003686 SmallPtrSetImpl<const PHINode*> &PHIs,
3687 unsigned CharSize) {
Eric Christopher4899cbc2010-03-05 06:58:57 +00003688 // Look through noop bitcast instructions.
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003689 V = V->stripPointerCasts();
Eric Christopher4899cbc2010-03-05 06:58:57 +00003690
3691 // If this is a PHI node, there are two cases: either we have already seen it
3692 // or we haven't.
Pete Cooper35b00d52016-08-13 01:05:32 +00003693 if (const PHINode *PN = dyn_cast<PHINode>(V)) {
David Blaikie70573dc2014-11-19 07:49:26 +00003694 if (!PHIs.insert(PN).second)
Eric Christopher4899cbc2010-03-05 06:58:57 +00003695 return ~0ULL; // already in the set.
3696
3697 // If it was new, see if all the input strings are the same length.
3698 uint64_t LenSoFar = ~0ULL;
Pete Cooper833f34d2015-05-12 20:05:31 +00003699 for (Value *IncValue : PN->incoming_values()) {
Matthias Braun50ec0b52017-05-19 22:37:09 +00003700 uint64_t Len = GetStringLengthH(IncValue, PHIs, CharSize);
Eric Christopher4899cbc2010-03-05 06:58:57 +00003701 if (Len == 0) return 0; // Unknown length -> unknown.
3702
3703 if (Len == ~0ULL) continue;
3704
3705 if (Len != LenSoFar && LenSoFar != ~0ULL)
3706 return 0; // Disagree -> unknown.
3707 LenSoFar = Len;
3708 }
3709
3710 // Success, all agree.
3711 return LenSoFar;
3712 }
3713
3714 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
Pete Cooper35b00d52016-08-13 01:05:32 +00003715 if (const SelectInst *SI = dyn_cast<SelectInst>(V)) {
Matthias Braun50ec0b52017-05-19 22:37:09 +00003716 uint64_t Len1 = GetStringLengthH(SI->getTrueValue(), PHIs, CharSize);
Eric Christopher4899cbc2010-03-05 06:58:57 +00003717 if (Len1 == 0) return 0;
Matthias Braun50ec0b52017-05-19 22:37:09 +00003718 uint64_t Len2 = GetStringLengthH(SI->getFalseValue(), PHIs, CharSize);
Eric Christopher4899cbc2010-03-05 06:58:57 +00003719 if (Len2 == 0) return 0;
3720 if (Len1 == ~0ULL) return Len2;
3721 if (Len2 == ~0ULL) return Len1;
3722 if (Len1 != Len2) return 0;
3723 return Len1;
3724 }
Craig Topper1bef2c82012-12-22 19:15:35 +00003725
Chris Lattnercf9e8f62012-02-05 02:29:43 +00003726 // Otherwise, see if we can read the string.
Matthias Braun50ec0b52017-05-19 22:37:09 +00003727 ConstantDataArraySlice Slice;
3728 if (!getConstantDataArrayInfo(V, Slice, CharSize))
Eric Christopher4899cbc2010-03-05 06:58:57 +00003729 return 0;
3730
Matthias Braun50ec0b52017-05-19 22:37:09 +00003731 if (Slice.Array == nullptr)
3732 return 1;
3733
3734 // Search for nul characters
3735 unsigned NullIndex = 0;
3736 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
3737 if (Slice.Array->getElementAsInteger(Slice.Offset + NullIndex) == 0)
3738 break;
3739 }
3740
3741 return NullIndex + 1;
Eric Christopher4899cbc2010-03-05 06:58:57 +00003742}
3743
Sanjay Patelaee84212014-11-04 16:27:42 +00003744/// If we can compute the length of the string pointed to by
Eric Christopher4899cbc2010-03-05 06:58:57 +00003745/// the specified pointer, return 'len+1'. If we can't, return 0.
David Bolvansky1f343fa2018-05-22 20:27:36 +00003746uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
David Bolvansky41f4b642018-05-22 15:41:23 +00003747 if (!V->getType()->isPointerTy())
3748 return 0;
Eric Christopher4899cbc2010-03-05 06:58:57 +00003749
Pete Cooper35b00d52016-08-13 01:05:32 +00003750 SmallPtrSet<const PHINode*, 32> PHIs;
Matthias Braun50ec0b52017-05-19 22:37:09 +00003751 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
Eric Christopher4899cbc2010-03-05 06:58:57 +00003752 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
3753 // an empty string as a length.
3754 return Len == ~0ULL ? 1 : Len;
3755}
Dan Gohmana4fcd242010-12-15 20:02:24 +00003756
Florian Hahnfd72bf22019-08-15 12:13:02 +00003757const Value *
3758llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
3759 bool MustPreserveNullness) {
Chandler Carruth363ac682019-01-07 05:42:51 +00003760 assert(Call &&
3761 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
3762 if (const Value *RV = Call->getReturnedArgOperand())
Piotr Padlewskid6f73462018-05-23 09:16:44 +00003763 return RV;
3764 // This can be used only as a aliasing property.
Florian Hahnfd72bf22019-08-15 12:13:02 +00003765 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
3766 Call, MustPreserveNullness))
Chandler Carruth363ac682019-01-07 05:42:51 +00003767 return Call->getArgOperand(0);
Piotr Padlewskid6f73462018-05-23 09:16:44 +00003768 return nullptr;
3769}
3770
3771bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
Florian Hahnfd72bf22019-08-15 12:13:02 +00003772 const CallBase *Call, bool MustPreserveNullness) {
Chandler Carruth363ac682019-01-07 05:42:51 +00003773 return Call->getIntrinsicID() == Intrinsic::launder_invariant_group ||
Evgeniy Stepanov50dc28b2019-07-03 20:19:14 +00003774 Call->getIntrinsicID() == Intrinsic::strip_invariant_group ||
Evgeniy Stepanovd752f5e2019-07-17 19:24:02 +00003775 Call->getIntrinsicID() == Intrinsic::aarch64_irg ||
Florian Hahn3f2850b2019-08-15 18:39:56 +00003776 Call->getIntrinsicID() == Intrinsic::aarch64_tagp ||
3777 (!MustPreserveNullness &&
3778 Call->getIntrinsicID() == Intrinsic::ptrmask);
Piotr Padlewskid6f73462018-05-23 09:16:44 +00003779}
3780
Adrian Prantl5f8f34e42018-05-01 15:54:18 +00003781/// \p PN defines a loop-variant pointer to an object. Check if the
Adam Nemete2b885c2015-04-23 20:09:20 +00003782/// previous iteration of the loop was referring to the same object as \p PN.
Pete Cooper35b00d52016-08-13 01:05:32 +00003783static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
3784 const LoopInfo *LI) {
Adam Nemete2b885c2015-04-23 20:09:20 +00003785 // Find the loop-defined value.
3786 Loop *L = LI->getLoopFor(PN->getParent());
3787 if (PN->getNumIncomingValues() != 2)
3788 return true;
3789
3790 // Find the value from previous iteration.
3791 auto *PrevValue = dyn_cast<Instruction>(PN->getIncomingValue(0));
3792 if (!PrevValue || LI->getLoopFor(PrevValue->getParent()) != L)
3793 PrevValue = dyn_cast<Instruction>(PN->getIncomingValue(1));
3794 if (!PrevValue || LI->getLoopFor(PrevValue->getParent()) != L)
3795 return true;
3796
3797 // If a new pointer is loaded in the loop, the pointer references a different
3798 // object in every iteration. E.g.:
3799 // for (i)
3800 // int *p = a[i];
3801 // ...
3802 if (auto *Load = dyn_cast<LoadInst>(PrevValue))
3803 if (!L->isLoopInvariant(Load->getPointerOperand()))
3804 return false;
3805 return true;
3806}
3807
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003808Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
3809 unsigned MaxLookup) {
Dan Gohmana4fcd242010-12-15 20:02:24 +00003810 if (!V->getType()->isPointerTy())
3811 return V;
3812 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
3813 if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
3814 V = GEP->getPointerOperand();
Matt Arsenault70f4db882014-07-15 00:56:40 +00003815 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
3816 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
Dan Gohmana4fcd242010-12-15 20:02:24 +00003817 V = cast<Operator>(V)->getOperand(0);
3818 } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) {
Sanjoy Das5ce32722016-04-08 00:48:30 +00003819 if (GA->isInterposable())
Dan Gohmana4fcd242010-12-15 20:02:24 +00003820 return V;
3821 V = GA->getAliasee();
Craig Topper85482412017-04-12 22:29:23 +00003822 } else if (isa<AllocaInst>(V)) {
3823 // An alloca can't be further simplified.
3824 return V;
Dan Gohmana4fcd242010-12-15 20:02:24 +00003825 } else {
Chandler Carruth363ac682019-01-07 05:42:51 +00003826 if (auto *Call = dyn_cast<CallBase>(V)) {
Piotr Padlewski5b3db452018-07-02 04:49:30 +00003827 // CaptureTracking can know about special capturing properties of some
3828 // intrinsics like launder.invariant.group, that can't be expressed with
3829 // the attributes, but have properties like returning aliasing pointer.
3830 // Because some analysis may assume that nocaptured pointer is not
3831 // returned from some special intrinsic (because function would have to
3832 // be marked with returns attribute), it is crucial to use this function
3833 // because it should be in sync with CaptureTracking. Not using it may
3834 // cause weird miscompilations where 2 aliasing pointers are assumed to
3835 // noalias.
Florian Hahn3f2850b2019-08-15 18:39:56 +00003836 if (auto *RP = getArgumentAliasingToReturnedPointer(Call, false)) {
Piotr Padlewskid6f73462018-05-23 09:16:44 +00003837 V = RP;
Hal Finkel5c12d8f2016-07-11 01:32:20 +00003838 continue;
3839 }
Piotr Padlewskid6f73462018-05-23 09:16:44 +00003840 }
Hal Finkel5c12d8f2016-07-11 01:32:20 +00003841
Dan Gohman05b18f12010-12-15 20:49:55 +00003842 // See if InstructionSimplify knows any relevant tricks.
3843 if (Instruction *I = dyn_cast<Instruction>(V))
Daniel Jasperaec2fa32016-12-19 08:22:17 +00003844 // TODO: Acquire a DominatorTree and AssumptionCache and use them.
Daniel Berlin4d0fe642017-04-28 19:55:38 +00003845 if (Value *Simplified = SimplifyInstruction(I, {DL, I})) {
Dan Gohman05b18f12010-12-15 20:49:55 +00003846 V = Simplified;
3847 continue;
3848 }
3849
Dan Gohmana4fcd242010-12-15 20:02:24 +00003850 return V;
3851 }
3852 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
3853 }
3854 return V;
3855}
Nick Lewycky3e334a42011-06-27 04:20:45 +00003856
Bjorn Pettersson71e8c6f2019-04-24 06:55:50 +00003857void llvm::GetUnderlyingObjects(const Value *V,
3858 SmallVectorImpl<const Value *> &Objects,
Adam Nemete2b885c2015-04-23 20:09:20 +00003859 const DataLayout &DL, LoopInfo *LI,
3860 unsigned MaxLookup) {
Bjorn Pettersson71e8c6f2019-04-24 06:55:50 +00003861 SmallPtrSet<const Value *, 4> Visited;
3862 SmallVector<const Value *, 4> Worklist;
Dan Gohmaned7c24e22012-05-10 18:57:38 +00003863 Worklist.push_back(V);
3864 do {
Bjorn Pettersson71e8c6f2019-04-24 06:55:50 +00003865 const Value *P = Worklist.pop_back_val();
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003866 P = GetUnderlyingObject(P, DL, MaxLookup);
Dan Gohmaned7c24e22012-05-10 18:57:38 +00003867
David Blaikie70573dc2014-11-19 07:49:26 +00003868 if (!Visited.insert(P).second)
Dan Gohmaned7c24e22012-05-10 18:57:38 +00003869 continue;
3870
Bjorn Pettersson71e8c6f2019-04-24 06:55:50 +00003871 if (auto *SI = dyn_cast<SelectInst>(P)) {
Dan Gohmaned7c24e22012-05-10 18:57:38 +00003872 Worklist.push_back(SI->getTrueValue());
3873 Worklist.push_back(SI->getFalseValue());
3874 continue;
3875 }
3876
Bjorn Pettersson71e8c6f2019-04-24 06:55:50 +00003877 if (auto *PN = dyn_cast<PHINode>(P)) {
Adam Nemete2b885c2015-04-23 20:09:20 +00003878 // If this PHI changes the underlying object in every iteration of the
3879 // loop, don't look through it. Consider:
3880 // int **A;
3881 // for (i) {
3882 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
3883 // Curr = A[i];
3884 // *Prev, *Curr;
3885 //
3886 // Prev is tracking Curr one iteration behind so they refer to different
3887 // underlying objects.
3888 if (!LI || !LI->isLoopHeader(PN->getParent()) ||
3889 isSameUnderlyingObjectInLoop(PN, LI))
Pete Cooper833f34d2015-05-12 20:05:31 +00003890 for (Value *IncValue : PN->incoming_values())
3891 Worklist.push_back(IncValue);
Dan Gohmaned7c24e22012-05-10 18:57:38 +00003892 continue;
3893 }
3894
3895 Objects.push_back(P);
3896 } while (!Worklist.empty());
3897}
3898
Hiroshi Inoueb9417db2017-08-01 03:32:15 +00003899/// This is the function that does the work of looking through basic
3900/// ptrtoint+arithmetic+inttoptr sequences.
3901static const Value *getUnderlyingObjectFromInt(const Value *V) {
3902 do {
3903 if (const Operator *U = dyn_cast<Operator>(V)) {
3904 // If we find a ptrtoint, we can transfer control back to the
3905 // regular getUnderlyingObjectFromInt.
3906 if (U->getOpcode() == Instruction::PtrToInt)
3907 return U->getOperand(0);
3908 // If we find an add of a constant, a multiplied value, or a phi, it's
3909 // likely that the other operand will lead us to the base
3910 // object. We don't have to worry about the case where the
3911 // object address is somehow being computed by the multiply,
3912 // because our callers only care when the result is an
3913 // identifiable object.
3914 if (U->getOpcode() != Instruction::Add ||
3915 (!isa<ConstantInt>(U->getOperand(1)) &&
3916 Operator::getOpcode(U->getOperand(1)) != Instruction::Mul &&
3917 !isa<PHINode>(U->getOperand(1))))
3918 return V;
3919 V = U->getOperand(0);
3920 } else {
3921 return V;
3922 }
3923 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
3924 } while (true);
3925}
3926
3927/// This is a wrapper around GetUnderlyingObjects and adds support for basic
3928/// ptrtoint+arithmetic+inttoptr sequences.
Hiroshi Inoueb49b0152017-10-12 06:26:04 +00003929/// It returns false if unidentified object is found in GetUnderlyingObjects.
3930bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
Hiroshi Inoueb9417db2017-08-01 03:32:15 +00003931 SmallVectorImpl<Value *> &Objects,
3932 const DataLayout &DL) {
3933 SmallPtrSet<const Value *, 16> Visited;
3934 SmallVector<const Value *, 4> Working(1, V);
3935 do {
3936 V = Working.pop_back_val();
3937
Bjorn Pettersson71e8c6f2019-04-24 06:55:50 +00003938 SmallVector<const Value *, 4> Objs;
3939 GetUnderlyingObjects(V, Objs, DL);
Hiroshi Inoueb9417db2017-08-01 03:32:15 +00003940
Bjorn Pettersson71e8c6f2019-04-24 06:55:50 +00003941 for (const Value *V : Objs) {
Hiroshi Inoueb9417db2017-08-01 03:32:15 +00003942 if (!Visited.insert(V).second)
3943 continue;
3944 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
3945 const Value *O =
3946 getUnderlyingObjectFromInt(cast<User>(V)->getOperand(0));
3947 if (O->getType()->isPointerTy()) {
3948 Working.push_back(O);
3949 continue;
3950 }
3951 }
Hiroshi Inoue0bd906e2017-08-02 18:16:32 +00003952 // If GetUnderlyingObjects fails to find an identifiable object,
3953 // getUnderlyingObjectsForCodeGen also fails for safety.
3954 if (!isIdentifiedObject(V)) {
3955 Objects.clear();
Hiroshi Inoueb49b0152017-10-12 06:26:04 +00003956 return false;
Hiroshi Inoue0bd906e2017-08-02 18:16:32 +00003957 }
Hiroshi Inoueb9417db2017-08-01 03:32:15 +00003958 Objects.push_back(const_cast<Value *>(V));
3959 }
3960 } while (!Working.empty());
Hiroshi Inoueb49b0152017-10-12 06:26:04 +00003961 return true;
Hiroshi Inoueb9417db2017-08-01 03:32:15 +00003962}
3963
Sanjay Patelaee84212014-11-04 16:27:42 +00003964/// Return true if the only users of this pointer are lifetime markers.
Nick Lewycky3e334a42011-06-27 04:20:45 +00003965bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
Chandler Carruthcdf47882014-03-09 03:16:01 +00003966 for (const User *U : V->users()) {
3967 const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
Nick Lewycky3e334a42011-06-27 04:20:45 +00003968 if (!II) return false;
3969
Vedant Kumarb264d692018-12-21 21:49:40 +00003970 if (!II->isLifetimeStartOrEnd())
Nick Lewycky3e334a42011-06-27 04:20:45 +00003971 return false;
3972 }
3973 return true;
3974}
Dan Gohman75d7d5e2011-12-14 23:49:11 +00003975
Philip Reames1e1db802019-09-10 21:12:29 +00003976bool llvm::mustSuppressSpeculation(const LoadInst &LI) {
3977 if (!LI.isUnordered())
3978 return true;
3979 const Function &F = *LI.getFunction();
3980 // Speculative load may create a race that did not exist in the source.
3981 return F.hasFnAttribute(Attribute::SanitizeThread) ||
3982 // Speculative load may load data from dirty regions.
3983 F.hasFnAttribute(Attribute::SanitizeAddress) ||
3984 F.hasFnAttribute(Attribute::SanitizeHWAddress);
3985}
3986
3987
Sanjoy Dasf8a0db52015-05-18 18:07:00 +00003988bool llvm::isSafeToSpeculativelyExecute(const Value *V,
3989 const Instruction *CtxI,
Sean Silva45835e72016-07-02 23:47:27 +00003990 const DominatorTree *DT) {
Dan Gohman7ac046a2012-01-04 23:01:09 +00003991 const Operator *Inst = dyn_cast<Operator>(V);
3992 if (!Inst)
3993 return false;
3994
Dan Gohman75d7d5e2011-12-14 23:49:11 +00003995 for (unsigned i = 0, e = Inst->getNumOperands(); i != e; ++i)
3996 if (Constant *C = dyn_cast<Constant>(Inst->getOperand(i)))
3997 if (C->canTrap())
3998 return false;
3999
4000 switch (Inst->getOpcode()) {
4001 default:
4002 return true;
4003 case Instruction::UDiv:
David Majnemerf20d7c42014-11-04 23:49:08 +00004004 case Instruction::URem: {
4005 // x / y is undefined if y == 0.
4006 const APInt *V;
4007 if (match(Inst->getOperand(1), m_APInt(V)))
4008 return *V != 0;
4009 return false;
4010 }
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004011 case Instruction::SDiv:
4012 case Instruction::SRem: {
David Majnemerf20d7c42014-11-04 23:49:08 +00004013 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
David Majnemer8a6578a2015-02-01 19:10:19 +00004014 const APInt *Numerator, *Denominator;
4015 if (!match(Inst->getOperand(1), m_APInt(Denominator)))
4016 return false;
4017 // We cannot hoist this division if the denominator is 0.
4018 if (*Denominator == 0)
4019 return false;
4020 // It's safe to hoist if the denominator is not 0 or -1.
4021 if (*Denominator != -1)
4022 return true;
4023 // At this point we know that the denominator is -1. It is safe to hoist as
4024 // long we know that the numerator is not INT_MIN.
4025 if (match(Inst->getOperand(0), m_APInt(Numerator)))
4026 return !Numerator->isMinSignedValue();
4027 // The numerator *might* be MinSignedValue.
David Majnemerf20d7c42014-11-04 23:49:08 +00004028 return false;
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004029 }
4030 case Instruction::Load: {
4031 const LoadInst *LI = cast<LoadInst>(Inst);
Philip Reames1e1db802019-09-10 21:12:29 +00004032 if (mustSuppressSpeculation(*LI))
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004033 return false;
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004034 const DataLayout &DL = LI->getModule()->getDataLayout();
Guillaume Chatelet301b4122019-10-21 15:10:26 +00004035 return isDereferenceableAndAlignedPointer(
4036 LI->getPointerOperand(), LI->getType(), MaybeAlign(LI->getAlignment()),
4037 DL, CtxI, DT);
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004038 }
Nick Lewyckyb4039f62011-12-21 05:52:02 +00004039 case Instruction::Call: {
Matt Arsenaultcf5e7fe2017-04-28 21:13:09 +00004040 auto *CI = cast<const CallInst>(Inst);
4041 const Function *Callee = CI->getCalledFunction();
David Majnemer0a92f862015-08-28 21:13:39 +00004042
Matt Arsenault6a288c12017-05-03 02:26:10 +00004043 // The called function could have undefined behavior or side-effects, even
4044 // if marked readnone nounwind.
4045 return Callee && Callee->isSpeculatable();
Nick Lewyckyb4039f62011-12-21 05:52:02 +00004046 }
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004047 case Instruction::VAArg:
4048 case Instruction::Alloca:
4049 case Instruction::Invoke:
Craig Topper784929d2019-02-08 20:48:56 +00004050 case Instruction::CallBr:
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004051 case Instruction::PHI:
4052 case Instruction::Store:
4053 case Instruction::Ret:
4054 case Instruction::Br:
4055 case Instruction::IndirectBr:
4056 case Instruction::Switch:
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004057 case Instruction::Unreachable:
4058 case Instruction::Fence:
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004059 case Instruction::AtomicRMW:
4060 case Instruction::AtomicCmpXchg:
David Majnemer654e1302015-07-31 17:58:14 +00004061 case Instruction::LandingPad:
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004062 case Instruction::Resume:
David Majnemer8a1c45d2015-12-12 05:38:55 +00004063 case Instruction::CatchSwitch:
David Majnemer654e1302015-07-31 17:58:14 +00004064 case Instruction::CatchPad:
David Majnemer654e1302015-07-31 17:58:14 +00004065 case Instruction::CatchRet:
4066 case Instruction::CleanupPad:
4067 case Instruction::CleanupRet:
Dan Gohman75d7d5e2011-12-14 23:49:11 +00004068 return false; // Misc instructions which have effects
4069 }
4070}
Dan Gohman1b0f79d2013-01-31 02:40:59 +00004071
Quentin Colombet6443cce2015-08-06 18:44:34 +00004072bool llvm::mayBeMemoryDependent(const Instruction &I) {
4073 return I.mayReadOrWriteMemory() || !isSafeToSpeculativelyExecute(&I);
4074}
4075
Nikita Popovd0f13e62019-05-26 13:22:01 +00004076/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
4077static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
4078 switch (OR) {
4079 case ConstantRange::OverflowResult::MayOverflow:
4080 return OverflowResult::MayOverflow;
Nikita Popov332c1002019-05-28 18:08:31 +00004081 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4082 return OverflowResult::AlwaysOverflowsLow;
4083 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4084 return OverflowResult::AlwaysOverflowsHigh;
Nikita Popovd0f13e62019-05-26 13:22:01 +00004085 case ConstantRange::OverflowResult::NeverOverflows:
4086 return OverflowResult::NeverOverflows;
4087 }
4088 llvm_unreachable("Unknown OverflowResult");
4089}
4090
4091/// Combine constant ranges from computeConstantRange() and computeKnownBits().
4092static ConstantRange computeConstantRangeIncludingKnownBits(
4093 const Value *V, bool ForSigned, const DataLayout &DL, unsigned Depth,
4094 AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT,
4095 OptimizationRemarkEmitter *ORE = nullptr, bool UseInstrInfo = true) {
4096 KnownBits Known = computeKnownBits(
4097 V, DL, Depth, AC, CxtI, DT, ORE, UseInstrInfo);
4098 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, ForSigned);
4099 ConstantRange CR2 = computeConstantRange(V, UseInstrInfo);
4100 ConstantRange::PreferredRangeType RangeType =
4101 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4102 return CR1.intersectWith(CR2, RangeType);
4103}
4104
Florian Hahn19f9e322018-08-17 14:39:04 +00004105OverflowResult llvm::computeOverflowForUnsignedMul(
4106 const Value *LHS, const Value *RHS, const DataLayout &DL,
4107 AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT,
4108 bool UseInstrInfo) {
Nikita Popovd0f13e62019-05-26 13:22:01 +00004109 KnownBits LHSKnown = computeKnownBits(LHS, DL, /*Depth=*/0, AC, CxtI, DT,
4110 nullptr, UseInstrInfo);
4111 KnownBits RHSKnown = computeKnownBits(RHS, DL, /*Depth=*/0, AC, CxtI, DT,
4112 nullptr, UseInstrInfo);
4113 ConstantRange LHSRange = ConstantRange::fromKnownBits(LHSKnown, false);
4114 ConstantRange RHSRange = ConstantRange::fromKnownBits(RHSKnown, false);
4115 return mapOverflowResult(LHSRange.unsignedMulMayOverflow(RHSRange));
David Majnemer491331a2015-01-02 07:29:43 +00004116}
David Majnemer5310c1e2015-01-07 00:39:50 +00004117
Florian Hahn19f9e322018-08-17 14:39:04 +00004118OverflowResult
4119llvm::computeOverflowForSignedMul(const Value *LHS, const Value *RHS,
4120 const DataLayout &DL, AssumptionCache *AC,
4121 const Instruction *CxtI,
4122 const DominatorTree *DT, bool UseInstrInfo) {
Omer Paparo Bivasfbb83de2018-05-10 19:46:19 +00004123 // Multiplying n * m significant bits yields a result of n + m significant
4124 // bits. If the total number of significant bits does not exceed the
4125 // result bit width (minus 1), there is no overflow.
4126 // This means if we have enough leading sign bits in the operands
4127 // we can guarantee that the result does not overflow.
4128 // Ref: "Hacker's Delight" by Henry Warren
4129 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
4130
4131 // Note that underestimating the number of sign bits gives a more
4132 // conservative answer.
4133 unsigned SignBits = ComputeNumSignBits(LHS, DL, 0, AC, CxtI, DT) +
4134 ComputeNumSignBits(RHS, DL, 0, AC, CxtI, DT);
4135
4136 // First handle the easy case: if we have enough sign bits there's
4137 // definitely no overflow.
4138 if (SignBits > BitWidth + 1)
4139 return OverflowResult::NeverOverflows;
4140
4141 // There are two ambiguous cases where there can be no overflow:
4142 // SignBits == BitWidth + 1 and
4143 // SignBits == BitWidth
4144 // The second case is difficult to check, therefore we only handle the
4145 // first case.
4146 if (SignBits == BitWidth + 1) {
4147 // It overflows only when both arguments are negative and the true
4148 // product is exactly the minimum negative number.
4149 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
4150 // For simplicity we just check if at least one side is not negative.
Florian Hahn19f9e322018-08-17 14:39:04 +00004151 KnownBits LHSKnown = computeKnownBits(LHS, DL, /*Depth=*/0, AC, CxtI, DT,
4152 nullptr, UseInstrInfo);
4153 KnownBits RHSKnown = computeKnownBits(RHS, DL, /*Depth=*/0, AC, CxtI, DT,
4154 nullptr, UseInstrInfo);
Omer Paparo Bivasfbb83de2018-05-10 19:46:19 +00004155 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
4156 return OverflowResult::NeverOverflows;
4157 }
4158 return OverflowResult::MayOverflow;
4159}
4160
Florian Hahn19f9e322018-08-17 14:39:04 +00004161OverflowResult llvm::computeOverflowForUnsignedAdd(
4162 const Value *LHS, const Value *RHS, const DataLayout &DL,
4163 AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT,
4164 bool UseInstrInfo) {
Nikita Popov20838192019-03-19 17:53:56 +00004165 ConstantRange LHSRange = computeConstantRangeIncludingKnownBits(
4166 LHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT,
4167 nullptr, UseInstrInfo);
4168 ConstantRange RHSRange = computeConstantRangeIncludingKnownBits(
4169 RHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT,
4170 nullptr, UseInstrInfo);
Nikita Popov614b1be2019-03-15 18:37:45 +00004171 return mapOverflowResult(LHSRange.unsignedAddMayOverflow(RHSRange));
David Majnemer5310c1e2015-01-07 00:39:50 +00004172}
James Molloy71b91c22015-05-11 14:42:20 +00004173
Pete Cooper35b00d52016-08-13 01:05:32 +00004174static OverflowResult computeOverflowForSignedAdd(const Value *LHS,
4175 const Value *RHS,
4176 const AddOperator *Add,
4177 const DataLayout &DL,
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004178 AssumptionCache *AC,
Pete Cooper35b00d52016-08-13 01:05:32 +00004179 const Instruction *CxtI,
4180 const DominatorTree *DT) {
Jingyue Wu10fcea52015-08-20 18:27:04 +00004181 if (Add && Add->hasNoSignedWrap()) {
4182 return OverflowResult::NeverOverflows;
4183 }
4184
Craig Topperbb973722017-05-15 02:44:08 +00004185 // If LHS and RHS each have at least two sign bits, the addition will look
4186 // like
4187 //
4188 // XX..... +
4189 // YY.....
4190 //
4191 // If the carry into the most significant position is 0, X and Y can't both
4192 // be 1 and therefore the carry out of the addition is also 0.
4193 //
4194 // If the carry into the most significant position is 1, X and Y can't both
4195 // be 0 and therefore the carry out of the addition is also 1.
4196 //
4197 // Since the carry into the most significant position is always equal to
4198 // the carry out of the addition, there is no signed overflow.
4199 if (ComputeNumSignBits(LHS, DL, 0, AC, CxtI, DT) > 1 &&
4200 ComputeNumSignBits(RHS, DL, 0, AC, CxtI, DT) > 1)
4201 return OverflowResult::NeverOverflows;
4202
Nikita Popov10edd2b2019-04-09 16:12:59 +00004203 ConstantRange LHSRange = computeConstantRangeIncludingKnownBits(
4204 LHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT);
4205 ConstantRange RHSRange = computeConstantRangeIncludingKnownBits(
4206 RHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT);
Nikita Popov322e2db2019-03-17 21:25:26 +00004207 OverflowResult OR =
4208 mapOverflowResult(LHSRange.signedAddMayOverflow(RHSRange));
4209 if (OR != OverflowResult::MayOverflow)
4210 return OR;
Jingyue Wu10fcea52015-08-20 18:27:04 +00004211
4212 // The remaining code needs Add to be available. Early returns if not so.
4213 if (!Add)
4214 return OverflowResult::MayOverflow;
4215
4216 // If the sign of Add is the same as at least one of the operands, this add
Nikita Popov280a6b02019-03-22 17:51:40 +00004217 // CANNOT overflow. If this can be determined from the known bits of the
4218 // operands the above signedAddMayOverflow() check will have already done so.
4219 // The only other way to improve on the known bits is from an assumption, so
4220 // call computeKnownBitsFromAssume() directly.
Jingyue Wu10fcea52015-08-20 18:27:04 +00004221 bool LHSOrRHSKnownNonNegative =
Nikita Popov6e9157d2019-04-09 07:13:09 +00004222 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
Fangrui Songf78650a2018-07-30 19:41:25 +00004223 bool LHSOrRHSKnownNegative =
Nikita Popov6e9157d2019-04-09 07:13:09 +00004224 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
Jingyue Wu10fcea52015-08-20 18:27:04 +00004225 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
Nikita Popov6e9157d2019-04-09 07:13:09 +00004226 KnownBits AddKnown(LHSRange.getBitWidth());
Nikita Popov280a6b02019-03-22 17:51:40 +00004227 computeKnownBitsFromAssume(
4228 Add, AddKnown, /*Depth=*/0, Query(DL, AC, CxtI, DT, true));
Craig Topper6e11a052017-05-08 16:22:48 +00004229 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
Nikita Popov280a6b02019-03-22 17:51:40 +00004230 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
Jingyue Wu10fcea52015-08-20 18:27:04 +00004231 return OverflowResult::NeverOverflows;
Jingyue Wu10fcea52015-08-20 18:27:04 +00004232 }
4233
4234 return OverflowResult::MayOverflow;
4235}
4236
Omer Paparo Bivasfbb83de2018-05-10 19:46:19 +00004237OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
4238 const Value *RHS,
4239 const DataLayout &DL,
4240 AssumptionCache *AC,
4241 const Instruction *CxtI,
4242 const DominatorTree *DT) {
Nikita Popov20838192019-03-19 17:53:56 +00004243 ConstantRange LHSRange = computeConstantRangeIncludingKnownBits(
4244 LHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT);
4245 ConstantRange RHSRange = computeConstantRangeIncludingKnownBits(
4246 RHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT);
Nikita Popov614b1be2019-03-15 18:37:45 +00004247 return mapOverflowResult(LHSRange.unsignedSubMayOverflow(RHSRange));
Omer Paparo Bivasfbb83de2018-05-10 19:46:19 +00004248}
4249
4250OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
4251 const Value *RHS,
4252 const DataLayout &DL,
4253 AssumptionCache *AC,
4254 const Instruction *CxtI,
4255 const DominatorTree *DT) {
4256 // If LHS and RHS each have at least two sign bits, the subtraction
4257 // cannot overflow.
4258 if (ComputeNumSignBits(LHS, DL, 0, AC, CxtI, DT) > 1 &&
4259 ComputeNumSignBits(RHS, DL, 0, AC, CxtI, DT) > 1)
4260 return OverflowResult::NeverOverflows;
4261
Nikita Popov4b2323d2019-04-09 17:01:49 +00004262 ConstantRange LHSRange = computeConstantRangeIncludingKnownBits(
4263 LHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT);
4264 ConstantRange RHSRange = computeConstantRangeIncludingKnownBits(
4265 RHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT);
Nikita Popov3af5b282019-03-21 17:23:51 +00004266 return mapOverflowResult(LHSRange.signedSubMayOverflow(RHSRange));
Omer Paparo Bivasfbb83de2018-05-10 19:46:19 +00004267}
4268
Nikita Popov79dffc62019-04-16 18:55:16 +00004269bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
Pete Cooper35b00d52016-08-13 01:05:32 +00004270 const DominatorTree &DT) {
Pete Cooper35b00d52016-08-13 01:05:32 +00004271 SmallVector<const BranchInst *, 2> GuardingBranches;
4272 SmallVector<const ExtractValueInst *, 2> Results;
Sanjoy Dasf49ca522016-05-29 00:34:42 +00004273
Nikita Popov79dffc62019-04-16 18:55:16 +00004274 for (const User *U : WO->users()) {
Pete Cooper35b00d52016-08-13 01:05:32 +00004275 if (const auto *EVI = dyn_cast<ExtractValueInst>(U)) {
Sanjoy Dasf49ca522016-05-29 00:34:42 +00004276 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
4277
4278 if (EVI->getIndices()[0] == 0)
4279 Results.push_back(EVI);
4280 else {
4281 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
4282
Pete Cooper35b00d52016-08-13 01:05:32 +00004283 for (const auto *U : EVI->users())
4284 if (const auto *B = dyn_cast<BranchInst>(U)) {
Sanjoy Dasf49ca522016-05-29 00:34:42 +00004285 assert(B->isConditional() && "How else is it using an i1?");
4286 GuardingBranches.push_back(B);
4287 }
4288 }
4289 } else {
4290 // We are using the aggregate directly in a way we don't want to analyze
4291 // here (storing it to a global, say).
4292 return false;
4293 }
4294 }
4295
Pete Cooper35b00d52016-08-13 01:05:32 +00004296 auto AllUsesGuardedByBranch = [&](const BranchInst *BI) {
Sanjoy Dasf49ca522016-05-29 00:34:42 +00004297 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(1));
4298 if (!NoWrapEdge.isSingleEdge())
4299 return false;
4300
4301 // Check if all users of the add are provably no-wrap.
Pete Cooper35b00d52016-08-13 01:05:32 +00004302 for (const auto *Result : Results) {
Sanjoy Dasf49ca522016-05-29 00:34:42 +00004303 // If the extractvalue itself is not executed on overflow, the we don't
4304 // need to check each use separately, since domination is transitive.
4305 if (DT.dominates(NoWrapEdge, Result->getParent()))
4306 continue;
4307
4308 for (auto &RU : Result->uses())
4309 if (!DT.dominates(NoWrapEdge, RU))
4310 return false;
4311 }
4312
4313 return true;
4314 };
4315
Eugene Zelenko75075ef2017-09-01 21:37:29 +00004316 return llvm::any_of(GuardingBranches, AllUsesGuardedByBranch);
Sanjoy Dasf49ca522016-05-29 00:34:42 +00004317}
4318
aqjune4187cb12019-11-07 01:12:17 +09004319bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V) {
4320 // If the value is a freeze instruction, then it can never
4321 // be undef or poison.
4322 if (isa<FreezeInst>(V))
4323 return true;
4324 // TODO: Some instructions are guaranteed to return neither undef
4325 // nor poison if their arguments are not poison/undef.
4326
4327 // TODO: Deal with other Constant subclasses.
4328 if (isa<ConstantInt>(V) || isa<GlobalVariable>(V))
4329 return true;
4330
4331 return false;
4332}
Sanjoy Dasf49ca522016-05-29 00:34:42 +00004333
Pete Cooper35b00d52016-08-13 01:05:32 +00004334OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
Jingyue Wu10fcea52015-08-20 18:27:04 +00004335 const DataLayout &DL,
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004336 AssumptionCache *AC,
Jingyue Wu10fcea52015-08-20 18:27:04 +00004337 const Instruction *CxtI,
4338 const DominatorTree *DT) {
4339 return ::computeOverflowForSignedAdd(Add->getOperand(0), Add->getOperand(1),
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004340 Add, DL, AC, CxtI, DT);
Jingyue Wu10fcea52015-08-20 18:27:04 +00004341}
4342
Pete Cooper35b00d52016-08-13 01:05:32 +00004343OverflowResult llvm::computeOverflowForSignedAdd(const Value *LHS,
4344 const Value *RHS,
Jingyue Wu10fcea52015-08-20 18:27:04 +00004345 const DataLayout &DL,
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004346 AssumptionCache *AC,
Jingyue Wu10fcea52015-08-20 18:27:04 +00004347 const Instruction *CxtI,
4348 const DominatorTree *DT) {
Daniel Jasperaec2fa32016-12-19 08:22:17 +00004349 return ::computeOverflowForSignedAdd(LHS, RHS, nullptr, DL, AC, CxtI, DT);
Jingyue Wu10fcea52015-08-20 18:27:04 +00004350}
4351
Jingyue Wu42f1d672015-07-28 18:22:40 +00004352bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
Hideto Ueno98d281a2019-07-29 13:35:34 +00004353 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
4354 // of time because it's possible for another thread to interfere with it for an
Eli Friedmanf1da33e2016-06-11 21:48:25 +00004355 // arbitrary length of time, but programs aren't allowed to rely on that.
Jingyue Wu42f1d672015-07-28 18:22:40 +00004356
Eli Friedmanf1da33e2016-06-11 21:48:25 +00004357 // If there is no successor, then execution can't transfer to it.
4358 if (const auto *CRI = dyn_cast<CleanupReturnInst>(I))
4359 return !CRI->unwindsToCaller();
4360 if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I))
4361 return !CatchSwitch->unwindsToCaller();
4362 if (isa<ResumeInst>(I))
4363 return false;
4364 if (isa<ReturnInst>(I))
4365 return false;
Sebastian Pop4a4d2452017-03-08 01:54:50 +00004366 if (isa<UnreachableInst>(I))
4367 return false;
Sanjoy Das9a65cd22016-06-08 17:48:36 +00004368
Eli Friedmanf1da33e2016-06-11 21:48:25 +00004369 // Calls can throw, or contain an infinite loop, or kill the process.
Sanjoy Das09455302016-12-31 22:12:31 +00004370 if (auto CS = ImmutableCallSite(I)) {
Sanjoy Das3bb2dbd2016-12-31 22:12:34 +00004371 // Call sites that throw have implicit non-local control flow.
4372 if (!CS.doesNotThrow())
4373 return false;
4374
Johannes Doerfert6ed459f2019-06-27 19:29:48 +00004375 // A function which doens't throw and has "willreturn" attribute will
4376 // always return.
4377 if (CS.hasFnAttr(Attribute::WillReturn))
4378 return true;
4379
Sanjoy Das3bb2dbd2016-12-31 22:12:34 +00004380 // Non-throwing call sites can loop infinitely, call exit/pthread_exit
4381 // etc. and thus not return. However, LLVM already assumes that
4382 //
4383 // - Thread exiting actions are modeled as writes to memory invisible to
4384 // the program.
4385 //
4386 // - Loops that don't have side effects (side effects are volatile/atomic
4387 // stores and IO) always terminate (see http://llvm.org/PR965).
4388 // Furthermore IO itself is also modeled as writes to memory invisible to
4389 // the program.
4390 //
4391 // We rely on those assumptions here, and use the memory effects of the call
4392 // target as a proxy for checking that it always returns.
4393
4394 // FIXME: This isn't aggressive enough; a call which only writes to a global
4395 // is guaranteed to return.
Hideto Ueno6e2be4e2019-07-30 18:35:29 +00004396 return CS.onlyReadsMemory() || CS.onlyAccessesArgMemory();
Eli Friedmanf1da33e2016-06-11 21:48:25 +00004397 }
4398
4399 // Other instructions return normally.
4400 return true;
Jingyue Wu42f1d672015-07-28 18:22:40 +00004401}
4402
Philip Reamesfbffd122018-03-08 21:25:30 +00004403bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
Hiroshi Inouec437f312019-01-30 05:26:31 +00004404 // TODO: This is slightly conservative for invoke instruction since exiting
Philip Reamesfbffd122018-03-08 21:25:30 +00004405 // via an exception *is* normal control for them.
4406 for (auto I = BB->begin(), E = BB->end(); I != E; ++I)
4407 if (!isGuaranteedToTransferExecutionToSuccessor(&*I))
4408 return false;
4409 return true;
4410}
4411
Jingyue Wu42f1d672015-07-28 18:22:40 +00004412bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
4413 const Loop *L) {
4414 // The loop header is guaranteed to be executed for every iteration.
4415 //
4416 // FIXME: Relax this constraint to cover all basic blocks that are
4417 // guaranteed to be executed at every iteration.
4418 if (I->getParent() != L->getHeader()) return false;
4419
4420 for (const Instruction &LI : *L->getHeader()) {
4421 if (&LI == I) return true;
4422 if (!isGuaranteedToTransferExecutionToSuccessor(&LI)) return false;
4423 }
4424 llvm_unreachable("Instruction not contained in its own parent basic block.");
4425}
4426
4427bool llvm::propagatesFullPoison(const Instruction *I) {
Nikita Popovad81d422019-06-13 19:45:36 +00004428 // TODO: This should include all instructions apart from phis, selects and
4429 // call-like instructions.
Jingyue Wu42f1d672015-07-28 18:22:40 +00004430 switch (I->getOpcode()) {
Sanjoy Das7b0b4082017-02-21 02:42:42 +00004431 case Instruction::Add:
4432 case Instruction::Sub:
4433 case Instruction::Xor:
4434 case Instruction::Trunc:
4435 case Instruction::BitCast:
4436 case Instruction::AddrSpaceCast:
Sanjoy Das5cd6c5ca2017-02-22 06:52:32 +00004437 case Instruction::Mul:
4438 case Instruction::Shl:
4439 case Instruction::GetElementPtr:
Sanjoy Das7b0b4082017-02-21 02:42:42 +00004440 // These operations all propagate poison unconditionally. Note that poison
4441 // is not any particular value, so xor or subtraction of poison with
4442 // itself still yields poison, not zero.
4443 return true;
Jingyue Wu42f1d672015-07-28 18:22:40 +00004444
Sanjoy Das7b0b4082017-02-21 02:42:42 +00004445 case Instruction::AShr:
4446 case Instruction::SExt:
4447 // For these operations, one bit of the input is replicated across
4448 // multiple output bits. A replicated poison bit is still poison.
4449 return true;
Jingyue Wu42f1d672015-07-28 18:22:40 +00004450
Sanjoy Das7b0b4082017-02-21 02:42:42 +00004451 case Instruction::ICmp:
4452 // Comparing poison with any value yields poison. This is why, for
4453 // instance, x s< (x +nsw 1) can be folded to true.
4454 return true;
Sanjoy Das70c2bbd2016-05-29 00:31:18 +00004455
Sanjoy Das7b0b4082017-02-21 02:42:42 +00004456 default:
4457 return false;
Jingyue Wu42f1d672015-07-28 18:22:40 +00004458 }
4459}
4460
4461const Value *llvm::getGuaranteedNonFullPoisonOp(const Instruction *I) {
4462 switch (I->getOpcode()) {
4463 case Instruction::Store:
4464 return cast<StoreInst>(I)->getPointerOperand();
4465
4466 case Instruction::Load:
4467 return cast<LoadInst>(I)->getPointerOperand();
4468
4469 case Instruction::AtomicCmpXchg:
4470 return cast<AtomicCmpXchgInst>(I)->getPointerOperand();
4471
4472 case Instruction::AtomicRMW:
4473 return cast<AtomicRMWInst>(I)->getPointerOperand();
4474
4475 case Instruction::UDiv:
4476 case Instruction::SDiv:
4477 case Instruction::URem:
4478 case Instruction::SRem:
4479 return I->getOperand(1);
4480
4481 default:
Philip Reames038e01d2019-06-13 19:27:56 +00004482 // Note: It's really tempting to think that a conditional branch or
4483 // switch should be listed here, but that's incorrect. It's not
4484 // branching off of poison which is UB, it is executing a side effecting
Johannes Doerfert3ed286a2019-07-11 01:14:48 +00004485 // instruction which follows the branch.
Jingyue Wu42f1d672015-07-28 18:22:40 +00004486 return nullptr;
4487 }
4488}
4489
Philip Reames4bf1c232019-06-10 20:41:27 +00004490bool llvm::mustTriggerUB(const Instruction *I,
4491 const SmallSet<const Value *, 16>& KnownPoison) {
4492 auto *NotPoison = getGuaranteedNonFullPoisonOp(I);
4493 return (NotPoison && KnownPoison.count(NotPoison));
4494}
4495
4496
Sanjoy Das08989c72017-04-30 19:41:19 +00004497bool llvm::programUndefinedIfFullPoison(const Instruction *PoisonI) {
Jingyue Wu42f1d672015-07-28 18:22:40 +00004498 // We currently only look for uses of poison values within the same basic
4499 // block, as that makes it easier to guarantee that the uses will be
4500 // executed given that PoisonI is executed.
4501 //
4502 // FIXME: Expand this to consider uses beyond the same basic block. To do
4503 // this, look out for the distinction between post-dominance and strong
4504 // post-dominance.
4505 const BasicBlock *BB = PoisonI->getParent();
4506
4507 // Set of instructions that we have proved will yield poison if PoisonI
4508 // does.
4509 SmallSet<const Value *, 16> YieldsPoison;
Sanjoy Dasa6155b62016-04-22 17:41:06 +00004510 SmallSet<const BasicBlock *, 4> Visited;
Jingyue Wu42f1d672015-07-28 18:22:40 +00004511 YieldsPoison.insert(PoisonI);
Sanjoy Dasa6155b62016-04-22 17:41:06 +00004512 Visited.insert(PoisonI->getParent());
Jingyue Wu42f1d672015-07-28 18:22:40 +00004513
Sanjoy Dasa6155b62016-04-22 17:41:06 +00004514 BasicBlock::const_iterator Begin = PoisonI->getIterator(), End = BB->end();
Jingyue Wu42f1d672015-07-28 18:22:40 +00004515
Sanjoy Dasa6155b62016-04-22 17:41:06 +00004516 unsigned Iter = 0;
4517 while (Iter++ < MaxDepth) {
4518 for (auto &I : make_range(Begin, End)) {
4519 if (&I != PoisonI) {
Philip Reames4bf1c232019-06-10 20:41:27 +00004520 if (mustTriggerUB(&I, YieldsPoison))
Sanjoy Dasa6155b62016-04-22 17:41:06 +00004521 return true;
4522 if (!isGuaranteedToTransferExecutionToSuccessor(&I))
4523 return false;
4524 }
4525
4526 // Mark poison that propagates from I through uses of I.
4527 if (YieldsPoison.count(&I)) {
4528 for (const User *User : I.users()) {
4529 const Instruction *UserI = cast<Instruction>(User);
4530 if (propagatesFullPoison(UserI))
4531 YieldsPoison.insert(User);
4532 }
Jingyue Wu42f1d672015-07-28 18:22:40 +00004533 }
4534 }
Sanjoy Dasa6155b62016-04-22 17:41:06 +00004535
4536 if (auto *NextBB = BB->getSingleSuccessor()) {
4537 if (Visited.insert(NextBB).second) {
4538 BB = NextBB;
4539 Begin = BB->getFirstNonPHI()->getIterator();
4540 End = BB->end();
4541 continue;
4542 }
4543 }
4544
4545 break;
Eugene Zelenko75075ef2017-09-01 21:37:29 +00004546 }
Jingyue Wu42f1d672015-07-28 18:22:40 +00004547 return false;
4548}
4549
Pete Cooper35b00d52016-08-13 01:05:32 +00004550static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
James Molloy134bec22015-08-11 09:12:57 +00004551 if (FMF.noNaNs())
4552 return true;
4553
4554 if (auto *C = dyn_cast<ConstantFP>(V))
4555 return !C->isNaN();
Thomas Livelyd47b5c72018-09-28 21:36:43 +00004556
4557 if (auto *C = dyn_cast<ConstantDataVector>(V)) {
4558 if (!C->getElementType()->isFloatingPointTy())
4559 return false;
4560 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
4561 if (C->getElementAsAPFloat(I).isNaN())
4562 return false;
4563 }
4564 return true;
4565 }
4566
James Molloy134bec22015-08-11 09:12:57 +00004567 return false;
4568}
4569
Pete Cooper35b00d52016-08-13 01:05:32 +00004570static bool isKnownNonZero(const Value *V) {
James Molloy134bec22015-08-11 09:12:57 +00004571 if (auto *C = dyn_cast<ConstantFP>(V))
4572 return !C->isZero();
Thomas Livelyd47b5c72018-09-28 21:36:43 +00004573
4574 if (auto *C = dyn_cast<ConstantDataVector>(V)) {
4575 if (!C->getElementType()->isFloatingPointTy())
4576 return false;
4577 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
4578 if (C->getElementAsAPFloat(I).isZero())
4579 return false;
4580 }
4581 return true;
4582 }
4583
James Molloy134bec22015-08-11 09:12:57 +00004584 return false;
4585}
4586
Nikolai Bozhenov1545eb32017-08-04 12:22:17 +00004587/// Match clamp pattern for float types without care about NaNs or signed zeros.
4588/// Given non-min/max outer cmp/select from the clamp pattern this
4589/// function recognizes if it can be substitued by a "canonical" min/max
4590/// pattern.
4591static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
4592 Value *CmpLHS, Value *CmpRHS,
4593 Value *TrueVal, Value *FalseVal,
4594 Value *&LHS, Value *&RHS) {
4595 // Try to match
4596 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
4597 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
4598 // and return description of the outer Max/Min.
4599
4600 // First, check if select has inverse order:
4601 if (CmpRHS == FalseVal) {
4602 std::swap(TrueVal, FalseVal);
4603 Pred = CmpInst::getInversePredicate(Pred);
4604 }
4605
4606 // Assume success now. If there's no match, callers should not use these anyway.
4607 LHS = TrueVal;
4608 RHS = FalseVal;
4609
4610 const APFloat *FC1;
4611 if (CmpRHS != TrueVal || !match(CmpRHS, m_APFloat(FC1)) || !FC1->isFinite())
4612 return {SPF_UNKNOWN, SPNB_NA, false};
4613
4614 const APFloat *FC2;
4615 switch (Pred) {
4616 case CmpInst::FCMP_OLT:
4617 case CmpInst::FCMP_OLE:
4618 case CmpInst::FCMP_ULT:
4619 case CmpInst::FCMP_ULE:
4620 if (match(FalseVal,
4621 m_CombineOr(m_OrdFMin(m_Specific(CmpLHS), m_APFloat(FC2)),
4622 m_UnordFMin(m_Specific(CmpLHS), m_APFloat(FC2)))) &&
4623 FC1->compare(*FC2) == APFloat::cmpResult::cmpLessThan)
4624 return {SPF_FMAXNUM, SPNB_RETURNS_ANY, false};
4625 break;
4626 case CmpInst::FCMP_OGT:
4627 case CmpInst::FCMP_OGE:
4628 case CmpInst::FCMP_UGT:
4629 case CmpInst::FCMP_UGE:
4630 if (match(FalseVal,
4631 m_CombineOr(m_OrdFMax(m_Specific(CmpLHS), m_APFloat(FC2)),
4632 m_UnordFMax(m_Specific(CmpLHS), m_APFloat(FC2)))) &&
4633 FC1->compare(*FC2) == APFloat::cmpResult::cmpGreaterThan)
4634 return {SPF_FMINNUM, SPNB_RETURNS_ANY, false};
4635 break;
4636 default:
4637 break;
4638 }
4639
4640 return {SPF_UNKNOWN, SPNB_NA, false};
4641}
4642
Artur Gainullinaf7ba8f2017-10-27 20:53:41 +00004643/// Recognize variations of:
4644/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
4645static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
4646 Value *CmpLHS, Value *CmpRHS,
4647 Value *TrueVal, Value *FalseVal) {
4648 // Swap the select operands and predicate to match the patterns below.
4649 if (CmpRHS != TrueVal) {
4650 Pred = ICmpInst::getSwappedPredicate(Pred);
4651 std::swap(TrueVal, FalseVal);
4652 }
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004653 const APInt *C1;
4654 if (CmpRHS == TrueVal && match(CmpRHS, m_APInt(C1))) {
4655 const APInt *C2;
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004656 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
4657 if (match(FalseVal, m_SMin(m_Specific(CmpLHS), m_APInt(C2))) &&
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004658 C1->slt(*C2) && Pred == CmpInst::ICMP_SLT)
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004659 return {SPF_SMAX, SPNB_NA, false};
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004660
4661 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
4662 if (match(FalseVal, m_SMax(m_Specific(CmpLHS), m_APInt(C2))) &&
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004663 C1->sgt(*C2) && Pred == CmpInst::ICMP_SGT)
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004664 return {SPF_SMIN, SPNB_NA, false};
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004665
4666 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
4667 if (match(FalseVal, m_UMin(m_Specific(CmpLHS), m_APInt(C2))) &&
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004668 C1->ult(*C2) && Pred == CmpInst::ICMP_ULT)
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004669 return {SPF_UMAX, SPNB_NA, false};
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004670
4671 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
4672 if (match(FalseVal, m_UMax(m_Specific(CmpLHS), m_APInt(C2))) &&
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004673 C1->ugt(*C2) && Pred == CmpInst::ICMP_UGT)
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004674 return {SPF_UMIN, SPNB_NA, false};
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004675 }
Artur Gainullinaf7ba8f2017-10-27 20:53:41 +00004676 return {SPF_UNKNOWN, SPNB_NA, false};
4677}
4678
Sanjay Patel78114302018-01-02 20:56:45 +00004679/// Recognize variations of:
4680/// a < c ? min(a,b) : min(b,c) ==> min(min(a,b),min(b,c))
4681static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
4682 Value *CmpLHS, Value *CmpRHS,
Sanjay Patel1d91ec32018-01-24 15:20:37 +00004683 Value *TVal, Value *FVal,
4684 unsigned Depth) {
Sanjay Patel78114302018-01-02 20:56:45 +00004685 // TODO: Allow FP min/max with nnan/nsz.
4686 assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
4687
Simon Pilgrimf62293e2019-09-23 13:15:52 +00004688 Value *A = nullptr, *B = nullptr;
Sanjay Patel1d91ec32018-01-24 15:20:37 +00004689 SelectPatternResult L = matchSelectPattern(TVal, A, B, nullptr, Depth + 1);
Sanjay Patel78114302018-01-02 20:56:45 +00004690 if (!SelectPatternResult::isMinOrMax(L.Flavor))
4691 return {SPF_UNKNOWN, SPNB_NA, false};
4692
Simon Pilgrimf62293e2019-09-23 13:15:52 +00004693 Value *C = nullptr, *D = nullptr;
Sanjay Patel1d91ec32018-01-24 15:20:37 +00004694 SelectPatternResult R = matchSelectPattern(FVal, C, D, nullptr, Depth + 1);
Sanjay Patel78114302018-01-02 20:56:45 +00004695 if (L.Flavor != R.Flavor)
4696 return {SPF_UNKNOWN, SPNB_NA, false};
4697
Sanjay Patele63d8dd2018-01-11 15:13:47 +00004698 // We have something like: x Pred y ? min(a, b) : min(c, d).
4699 // Try to match the compare to the min/max operations of the select operands.
4700 // First, make sure we have the right compare predicate.
Sanjay Patel78114302018-01-02 20:56:45 +00004701 switch (L.Flavor) {
4702 case SPF_SMIN:
4703 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
4704 Pred = ICmpInst::getSwappedPredicate(Pred);
4705 std::swap(CmpLHS, CmpRHS);
4706 }
4707 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
4708 break;
4709 return {SPF_UNKNOWN, SPNB_NA, false};
4710 case SPF_SMAX:
4711 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) {
4712 Pred = ICmpInst::getSwappedPredicate(Pred);
4713 std::swap(CmpLHS, CmpRHS);
4714 }
4715 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)
4716 break;
4717 return {SPF_UNKNOWN, SPNB_NA, false};
4718 case SPF_UMIN:
4719 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
4720 Pred = ICmpInst::getSwappedPredicate(Pred);
4721 std::swap(CmpLHS, CmpRHS);
4722 }
4723 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
4724 break;
4725 return {SPF_UNKNOWN, SPNB_NA, false};
4726 case SPF_UMAX:
4727 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
4728 Pred = ICmpInst::getSwappedPredicate(Pred);
4729 std::swap(CmpLHS, CmpRHS);
4730 }
4731 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
4732 break;
4733 return {SPF_UNKNOWN, SPNB_NA, false};
4734 default:
Sanjay Patel7dfe96a2018-01-08 18:31:13 +00004735 return {SPF_UNKNOWN, SPNB_NA, false};
Sanjay Patel78114302018-01-02 20:56:45 +00004736 }
4737
Sanjay Patele63d8dd2018-01-11 15:13:47 +00004738 // If there is a common operand in the already matched min/max and the other
4739 // min/max operands match the compare operands (either directly or inverted),
4740 // then this is min/max of the same flavor.
4741
Sanjay Patel78114302018-01-02 20:56:45 +00004742 // a pred c ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
Sanjay Patele63d8dd2018-01-11 15:13:47 +00004743 // ~c pred ~a ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
4744 if (D == B) {
4745 if ((CmpLHS == A && CmpRHS == C) || (match(C, m_Not(m_Specific(CmpLHS))) &&
4746 match(A, m_Not(m_Specific(CmpRHS)))))
4747 return {L.Flavor, SPNB_NA, false};
4748 }
Sanjay Patel78114302018-01-02 20:56:45 +00004749 // a pred d ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
Sanjay Patele63d8dd2018-01-11 15:13:47 +00004750 // ~d pred ~a ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
4751 if (C == B) {
4752 if ((CmpLHS == A && CmpRHS == D) || (match(D, m_Not(m_Specific(CmpLHS))) &&
4753 match(A, m_Not(m_Specific(CmpRHS)))))
4754 return {L.Flavor, SPNB_NA, false};
4755 }
Sanjay Patel78114302018-01-02 20:56:45 +00004756 // b pred c ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
Sanjay Patele63d8dd2018-01-11 15:13:47 +00004757 // ~c pred ~b ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
4758 if (D == A) {
4759 if ((CmpLHS == B && CmpRHS == C) || (match(C, m_Not(m_Specific(CmpLHS))) &&
4760 match(B, m_Not(m_Specific(CmpRHS)))))
4761 return {L.Flavor, SPNB_NA, false};
4762 }
Sanjay Patel78114302018-01-02 20:56:45 +00004763 // b pred d ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
Sanjay Patele63d8dd2018-01-11 15:13:47 +00004764 // ~d pred ~b ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
4765 if (C == A) {
4766 if ((CmpLHS == B && CmpRHS == D) || (match(D, m_Not(m_Specific(CmpLHS))) &&
4767 match(B, m_Not(m_Specific(CmpRHS)))))
4768 return {L.Flavor, SPNB_NA, false};
4769 }
Sanjay Patel78114302018-01-02 20:56:45 +00004770
4771 return {SPF_UNKNOWN, SPNB_NA, false};
4772}
4773
Artur Gainullinaf7ba8f2017-10-27 20:53:41 +00004774/// Match non-obvious integer minimum and maximum sequences.
4775static SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
4776 Value *CmpLHS, Value *CmpRHS,
4777 Value *TrueVal, Value *FalseVal,
Sanjay Patel1d91ec32018-01-24 15:20:37 +00004778 Value *&LHS, Value *&RHS,
4779 unsigned Depth) {
Artur Gainullinaf7ba8f2017-10-27 20:53:41 +00004780 // Assume success. If there's no match, callers should not use these anyway.
4781 LHS = TrueVal;
4782 RHS = FalseVal;
4783
4784 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
4785 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
4786 return SPR;
Sanjay Patel0c1c70a2017-01-20 22:18:47 +00004787
Sanjay Patel1d91ec32018-01-24 15:20:37 +00004788 SPR = matchMinMaxOfMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, Depth);
Sanjay Patel78114302018-01-02 20:56:45 +00004789 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
4790 return SPR;
Fangrui Songf78650a2018-07-30 19:41:25 +00004791
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004792 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
Sanjay Patel819f0962016-11-13 19:30:19 +00004793 return {SPF_UNKNOWN, SPNB_NA, false};
4794
Sanjay Patelcfcc42b2016-11-13 20:04:52 +00004795 // Z = X -nsw Y
4796 // (X >s Y) ? 0 : Z ==> (Z >s 0) ? 0 : Z ==> SMIN(Z, 0)
4797 // (X <s Y) ? 0 : Z ==> (Z <s 0) ? 0 : Z ==> SMAX(Z, 0)
4798 if (match(TrueVal, m_Zero()) &&
Sanjay Patel24c6f882017-01-21 17:51:25 +00004799 match(FalseVal, m_NSWSub(m_Specific(CmpLHS), m_Specific(CmpRHS))))
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004800 return {Pred == CmpInst::ICMP_SGT ? SPF_SMIN : SPF_SMAX, SPNB_NA, false};
Sanjay Patelcfcc42b2016-11-13 20:04:52 +00004801
4802 // Z = X -nsw Y
4803 // (X >s Y) ? Z : 0 ==> (Z >s 0) ? Z : 0 ==> SMAX(Z, 0)
4804 // (X <s Y) ? Z : 0 ==> (Z <s 0) ? Z : 0 ==> SMIN(Z, 0)
4805 if (match(FalseVal, m_Zero()) &&
Sanjay Patel24c6f882017-01-21 17:51:25 +00004806 match(TrueVal, m_NSWSub(m_Specific(CmpLHS), m_Specific(CmpRHS))))
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004807 return {Pred == CmpInst::ICMP_SGT ? SPF_SMAX : SPF_SMIN, SPNB_NA, false};
Sanjay Patelcfcc42b2016-11-13 20:04:52 +00004808
Artur Gainullinaf7ba8f2017-10-27 20:53:41 +00004809 const APInt *C1;
Sanjay Patel819f0962016-11-13 19:30:19 +00004810 if (!match(CmpRHS, m_APInt(C1)))
4811 return {SPF_UNKNOWN, SPNB_NA, false};
4812
4813 // An unsigned min/max can be written with a signed compare.
4814 const APInt *C2;
4815 if ((CmpLHS == TrueVal && match(FalseVal, m_APInt(C2))) ||
4816 (CmpLHS == FalseVal && match(TrueVal, m_APInt(C2)))) {
4817 // Is the sign bit set?
4818 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
4819 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
Craig Topper81d772c2017-11-08 19:38:45 +00004820 if (Pred == CmpInst::ICMP_SLT && C1->isNullValue() &&
4821 C2->isMaxSignedValue())
Sanjay Patel819f0962016-11-13 19:30:19 +00004822 return {CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false};
Sanjay Patel819f0962016-11-13 19:30:19 +00004823
4824 // Is the sign bit clear?
4825 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
4826 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004827 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnesValue() &&
4828 C2->isMinSignedValue())
Sanjay Patel819f0962016-11-13 19:30:19 +00004829 return {CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, SPNB_NA, false};
Sanjay Patel819f0962016-11-13 19:30:19 +00004830 }
4831
4832 // Look through 'not' ops to find disguised signed min/max.
4833 // (X >s C) ? ~X : ~C ==> (~X <s ~C) ? ~X : ~C ==> SMIN(~X, ~C)
4834 // (X <s C) ? ~X : ~C ==> (~X >s ~C) ? ~X : ~C ==> SMAX(~X, ~C)
4835 if (match(TrueVal, m_Not(m_Specific(CmpLHS))) &&
Sanjay Patel24c6f882017-01-21 17:51:25 +00004836 match(FalseVal, m_APInt(C2)) && ~(*C1) == *C2)
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004837 return {Pred == CmpInst::ICMP_SGT ? SPF_SMIN : SPF_SMAX, SPNB_NA, false};
Sanjay Patel819f0962016-11-13 19:30:19 +00004838
4839 // (X >s C) ? ~C : ~X ==> (~X <s ~C) ? ~C : ~X ==> SMAX(~C, ~X)
4840 // (X <s C) ? ~C : ~X ==> (~X >s ~C) ? ~C : ~X ==> SMIN(~C, ~X)
4841 if (match(FalseVal, m_Not(m_Specific(CmpLHS))) &&
Sanjay Patel24c6f882017-01-21 17:51:25 +00004842 match(TrueVal, m_APInt(C2)) && ~(*C1) == *C2)
Nikolai Bozhenov8dcab542017-10-19 15:36:18 +00004843 return {Pred == CmpInst::ICMP_SGT ? SPF_SMAX : SPF_SMIN, SPNB_NA, false};
Sanjay Patel819f0962016-11-13 19:30:19 +00004844
4845 return {SPF_UNKNOWN, SPNB_NA, false};
4846}
4847
Chen Zheng69bb0642018-07-21 12:27:54 +00004848bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW) {
Chen Zhengfdf13ef2018-07-12 03:06:04 +00004849 assert(X && Y && "Invalid operand");
4850
Chen Zheng69bb0642018-07-21 12:27:54 +00004851 // X = sub (0, Y) || X = sub nsw (0, Y)
4852 if ((!NeedNSW && match(X, m_Sub(m_ZeroInt(), m_Specific(Y)))) ||
4853 (NeedNSW && match(X, m_NSWSub(m_ZeroInt(), m_Specific(Y)))))
Chen Zhengfdf13ef2018-07-12 03:06:04 +00004854 return true;
4855
Chen Zheng69bb0642018-07-21 12:27:54 +00004856 // Y = sub (0, X) || Y = sub nsw (0, X)
4857 if ((!NeedNSW && match(Y, m_Sub(m_ZeroInt(), m_Specific(X)))) ||
4858 (NeedNSW && match(Y, m_NSWSub(m_ZeroInt(), m_Specific(X)))))
Chen Zhengfdf13ef2018-07-12 03:06:04 +00004859 return true;
4860
Chen Zheng69bb0642018-07-21 12:27:54 +00004861 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
Chen Zhengfdf13ef2018-07-12 03:06:04 +00004862 Value *A, *B;
Chen Zheng69bb0642018-07-21 12:27:54 +00004863 return (!NeedNSW && (match(X, m_Sub(m_Value(A), m_Value(B))) &&
4864 match(Y, m_Sub(m_Specific(B), m_Specific(A))))) ||
4865 (NeedNSW && (match(X, m_NSWSub(m_Value(A), m_Value(B))) &&
4866 match(Y, m_NSWSub(m_Specific(B), m_Specific(A)))));
Chen Zhengfdf13ef2018-07-12 03:06:04 +00004867}
4868
James Molloy134bec22015-08-11 09:12:57 +00004869static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
4870 FastMathFlags FMF,
James Molloy270ef8c2015-05-15 16:04:50 +00004871 Value *CmpLHS, Value *CmpRHS,
4872 Value *TrueVal, Value *FalseVal,
Sanjay Patel1d91ec32018-01-24 15:20:37 +00004873 Value *&LHS, Value *&RHS,
4874 unsigned Depth) {
Sanjay Patele7c94ef2018-11-04 14:28:48 +00004875 if (CmpInst::isFPPredicate(Pred)) {
4876 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
4877 // 0.0 operand, set the compare's 0.0 operands to that same value for the
4878 // purpose of identifying min/max. Disregard vector constants with undefined
4879 // elements because those can not be back-propagated for analysis.
4880 Value *OutputZeroVal = nullptr;
4881 if (match(TrueVal, m_AnyZeroFP()) && !match(FalseVal, m_AnyZeroFP()) &&
4882 !cast<Constant>(TrueVal)->containsUndefElement())
4883 OutputZeroVal = TrueVal;
4884 else if (match(FalseVal, m_AnyZeroFP()) && !match(TrueVal, m_AnyZeroFP()) &&
4885 !cast<Constant>(FalseVal)->containsUndefElement())
4886 OutputZeroVal = FalseVal;
4887
4888 if (OutputZeroVal) {
4889 if (match(CmpLHS, m_AnyZeroFP()))
4890 CmpLHS = OutputZeroVal;
4891 if (match(CmpRHS, m_AnyZeroFP()))
4892 CmpRHS = OutputZeroVal;
4893 }
4894 }
4895
James Molloy71b91c22015-05-11 14:42:20 +00004896 LHS = CmpLHS;
4897 RHS = CmpRHS;
4898
Sanjay Patel9a399792017-12-26 15:09:19 +00004899 // Signed zero may return inconsistent results between implementations.
4900 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
4901 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
4902 // Therefore, we behave conservatively and only proceed if at least one of the
4903 // operands is known to not be zero or if we don't care about signed zero.
James Molloy134bec22015-08-11 09:12:57 +00004904 switch (Pred) {
4905 default: break;
Sanjay Patel9a399792017-12-26 15:09:19 +00004906 // FIXME: Include OGT/OLT/UGT/ULT.
James Molloy134bec22015-08-11 09:12:57 +00004907 case CmpInst::FCMP_OGE: case CmpInst::FCMP_OLE:
4908 case CmpInst::FCMP_UGE: case CmpInst::FCMP_ULE:
4909 if (!FMF.noSignedZeros() && !isKnownNonZero(CmpLHS) &&
4910 !isKnownNonZero(CmpRHS))
4911 return {SPF_UNKNOWN, SPNB_NA, false};
4912 }
4913
4914 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
4915 bool Ordered = false;
4916
4917 // When given one NaN and one non-NaN input:
4918 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
4919 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
4920 // ordered comparison fails), which could be NaN or non-NaN.
4921 // so here we discover exactly what NaN behavior is required/accepted.
4922 if (CmpInst::isFPPredicate(Pred)) {
4923 bool LHSSafe = isKnownNonNaN(CmpLHS, FMF);
4924 bool RHSSafe = isKnownNonNaN(CmpRHS, FMF);
4925
4926 if (LHSSafe && RHSSafe) {
4927 // Both operands are known non-NaN.
4928 NaNBehavior = SPNB_RETURNS_ANY;
4929 } else if (CmpInst::isOrdered(Pred)) {
4930 // An ordered comparison will return false when given a NaN, so it
4931 // returns the RHS.
4932 Ordered = true;
4933 if (LHSSafe)
James Molloy8990b062015-08-12 15:11:43 +00004934 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
James Molloy134bec22015-08-11 09:12:57 +00004935 NaNBehavior = SPNB_RETURNS_NAN;
4936 else if (RHSSafe)
4937 NaNBehavior = SPNB_RETURNS_OTHER;
4938 else
4939 // Completely unsafe.
4940 return {SPF_UNKNOWN, SPNB_NA, false};
4941 } else {
4942 Ordered = false;
4943 // An unordered comparison will return true when given a NaN, so it
4944 // returns the LHS.
4945 if (LHSSafe)
James Molloy8990b062015-08-12 15:11:43 +00004946 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
James Molloy134bec22015-08-11 09:12:57 +00004947 NaNBehavior = SPNB_RETURNS_OTHER;
4948 else if (RHSSafe)
4949 NaNBehavior = SPNB_RETURNS_NAN;
4950 else
4951 // Completely unsafe.
4952 return {SPF_UNKNOWN, SPNB_NA, false};
James Molloy71b91c22015-05-11 14:42:20 +00004953 }
4954 }
4955
James Molloy71b91c22015-05-11 14:42:20 +00004956 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
James Molloy134bec22015-08-11 09:12:57 +00004957 std::swap(CmpLHS, CmpRHS);
4958 Pred = CmpInst::getSwappedPredicate(Pred);
4959 if (NaNBehavior == SPNB_RETURNS_NAN)
4960 NaNBehavior = SPNB_RETURNS_OTHER;
4961 else if (NaNBehavior == SPNB_RETURNS_OTHER)
4962 NaNBehavior = SPNB_RETURNS_NAN;
4963 Ordered = !Ordered;
4964 }
4965
4966 // ([if]cmp X, Y) ? X : Y
4967 if (TrueVal == CmpLHS && FalseVal == CmpRHS) {
James Molloy71b91c22015-05-11 14:42:20 +00004968 switch (Pred) {
James Molloy134bec22015-08-11 09:12:57 +00004969 default: return {SPF_UNKNOWN, SPNB_NA, false}; // Equality.
James Molloy71b91c22015-05-11 14:42:20 +00004970 case ICmpInst::ICMP_UGT:
James Molloy134bec22015-08-11 09:12:57 +00004971 case ICmpInst::ICMP_UGE: return {SPF_UMAX, SPNB_NA, false};
James Molloy71b91c22015-05-11 14:42:20 +00004972 case ICmpInst::ICMP_SGT:
James Molloy134bec22015-08-11 09:12:57 +00004973 case ICmpInst::ICMP_SGE: return {SPF_SMAX, SPNB_NA, false};
James Molloy71b91c22015-05-11 14:42:20 +00004974 case ICmpInst::ICMP_ULT:
James Molloy134bec22015-08-11 09:12:57 +00004975 case ICmpInst::ICMP_ULE: return {SPF_UMIN, SPNB_NA, false};
James Molloy71b91c22015-05-11 14:42:20 +00004976 case ICmpInst::ICMP_SLT:
James Molloy134bec22015-08-11 09:12:57 +00004977 case ICmpInst::ICMP_SLE: return {SPF_SMIN, SPNB_NA, false};
4978 case FCmpInst::FCMP_UGT:
4979 case FCmpInst::FCMP_UGE:
4980 case FCmpInst::FCMP_OGT:
4981 case FCmpInst::FCMP_OGE: return {SPF_FMAXNUM, NaNBehavior, Ordered};
4982 case FCmpInst::FCMP_ULT:
4983 case FCmpInst::FCMP_ULE:
4984 case FCmpInst::FCMP_OLT:
4985 case FCmpInst::FCMP_OLE: return {SPF_FMINNUM, NaNBehavior, Ordered};
James Molloy71b91c22015-05-11 14:42:20 +00004986 }
4987 }
Fangrui Songf78650a2018-07-30 19:41:25 +00004988
Chen Zhengccc84222018-07-16 02:23:00 +00004989 if (isKnownNegation(TrueVal, FalseVal)) {
4990 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
4991 // match against either LHS or sext(LHS).
4992 auto MaybeSExtCmpLHS =
4993 m_CombineOr(m_Specific(CmpLHS), m_SExt(m_Specific(CmpLHS)));
4994 auto ZeroOrAllOnes = m_CombineOr(m_ZeroInt(), m_AllOnes());
4995 auto ZeroOrOne = m_CombineOr(m_ZeroInt(), m_One());
4996 if (match(TrueVal, MaybeSExtCmpLHS)) {
4997 // Set the return values. If the compare uses the negated value (-X >s 0),
4998 // swap the return values because the negated value is always 'RHS'.
Sanjay Patel284ba0c2018-07-02 14:43:40 +00004999 LHS = TrueVal;
5000 RHS = FalseVal;
Chen Zhengccc84222018-07-16 02:23:00 +00005001 if (match(CmpLHS, m_Neg(m_Specific(FalseVal))))
5002 std::swap(LHS, RHS);
5003
5004 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
5005 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
5006 if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, ZeroOrAllOnes))
5007 return {SPF_ABS, SPNB_NA, false};
5008
Simon Pilgrima56f2822019-03-19 16:24:55 +00005009 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
5010 if (Pred == ICmpInst::ICMP_SGE && match(CmpRHS, ZeroOrOne))
5011 return {SPF_ABS, SPNB_NA, false};
5012
Chen Zhengccc84222018-07-16 02:23:00 +00005013 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
5014 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
5015 if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, ZeroOrOne))
5016 return {SPF_NABS, SPNB_NA, false};
5017 }
5018 else if (match(FalseVal, MaybeSExtCmpLHS)) {
5019 // Set the return values. If the compare uses the negated value (-X >s 0),
5020 // swap the return values because the negated value is always 'RHS'.
Sanjay Patel284ba0c2018-07-02 14:43:40 +00005021 LHS = FalseVal;
5022 RHS = TrueVal;
Chen Zhengccc84222018-07-16 02:23:00 +00005023 if (match(CmpLHS, m_Neg(m_Specific(TrueVal))))
5024 std::swap(LHS, RHS);
5025
5026 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
5027 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
5028 if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, ZeroOrAllOnes))
5029 return {SPF_NABS, SPNB_NA, false};
5030
5031 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
5032 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
5033 if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, ZeroOrOne))
5034 return {SPF_ABS, SPNB_NA, false};
James Molloy71b91c22015-05-11 14:42:20 +00005035 }
James Molloy71b91c22015-05-11 14:42:20 +00005036 }
5037
Nikolai Bozhenov1545eb32017-08-04 12:22:17 +00005038 if (CmpInst::isIntPredicate(Pred))
Sanjay Patel1d91ec32018-01-24 15:20:37 +00005039 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth);
Nikolai Bozhenov1545eb32017-08-04 12:22:17 +00005040
5041 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
5042 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
5043 // semantics than minNum. Be conservative in such case.
5044 if (NaNBehavior != SPNB_RETURNS_ANY ||
5045 (!FMF.noSignedZeros() && !isKnownNonZero(CmpLHS) &&
5046 !isKnownNonZero(CmpRHS)))
5047 return {SPF_UNKNOWN, SPNB_NA, false};
5048
5049 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
James Molloy71b91c22015-05-11 14:42:20 +00005050}
James Molloy270ef8c2015-05-15 16:04:50 +00005051
Nikolai Bozhenov74c047e2017-10-18 09:28:09 +00005052/// Helps to match a select pattern in case of a type mismatch.
5053///
5054/// The function processes the case when type of true and false values of a
5055/// select instruction differs from type of the cmp instruction operands because
Vedant Kumar1a8456d2018-03-02 18:57:02 +00005056/// of a cast instruction. The function checks if it is legal to move the cast
Nikolai Bozhenov74c047e2017-10-18 09:28:09 +00005057/// operation after "select". If yes, it returns the new second value of
5058/// "select" (with the assumption that cast is moved):
5059/// 1. As operand of cast instruction when both values of "select" are same cast
5060/// instructions.
5061/// 2. As restored constant (by applying reverse cast operation) when the first
5062/// value of the "select" is a cast operation and the second value is a
5063/// constant.
5064/// NOTE: We return only the new second value because the first value could be
5065/// accessed as operand of cast instruction.
James Molloy569cea62015-09-02 17:25:25 +00005066static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
5067 Instruction::CastOps *CastOp) {
Sanjay Patel14a4b812017-01-29 16:34:57 +00005068 auto *Cast1 = dyn_cast<CastInst>(V1);
5069 if (!Cast1)
James Molloy270ef8c2015-05-15 16:04:50 +00005070 return nullptr;
James Molloy270ef8c2015-05-15 16:04:50 +00005071
Sanjay Patel14a4b812017-01-29 16:34:57 +00005072 *CastOp = Cast1->getOpcode();
5073 Type *SrcTy = Cast1->getSrcTy();
5074 if (auto *Cast2 = dyn_cast<CastInst>(V2)) {
5075 // If V1 and V2 are both the same cast from the same type, look through V1.
5076 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
5077 return Cast2->getOperand(0);
James Molloy569cea62015-09-02 17:25:25 +00005078 return nullptr;
5079 }
5080
Sanjay Patel14a4b812017-01-29 16:34:57 +00005081 auto *C = dyn_cast<Constant>(V2);
5082 if (!C)
5083 return nullptr;
5084
David Majnemerd2a074b2016-04-29 18:40:34 +00005085 Constant *CastedTo = nullptr;
Sanjay Patel14a4b812017-01-29 16:34:57 +00005086 switch (*CastOp) {
5087 case Instruction::ZExt:
5088 if (CmpI->isUnsigned())
5089 CastedTo = ConstantExpr::getTrunc(C, SrcTy);
5090 break;
5091 case Instruction::SExt:
5092 if (CmpI->isSigned())
5093 CastedTo = ConstantExpr::getTrunc(C, SrcTy, true);
5094 break;
5095 case Instruction::Trunc:
Nikolai Bozhenov74c047e2017-10-18 09:28:09 +00005096 Constant *CmpConst;
Nikolai Bozhenov9723f122017-10-18 14:24:50 +00005097 if (match(CmpI->getOperand(1), m_Constant(CmpConst)) &&
5098 CmpConst->getType() == SrcTy) {
Nikolai Bozhenov74c047e2017-10-18 09:28:09 +00005099 // Here we have the following case:
5100 //
5101 // %cond = cmp iN %x, CmpConst
5102 // %tr = trunc iN %x to iK
5103 // %narrowsel = select i1 %cond, iK %t, iK C
5104 //
5105 // We can always move trunc after select operation:
5106 //
5107 // %cond = cmp iN %x, CmpConst
5108 // %widesel = select i1 %cond, iN %x, iN CmpConst
5109 // %tr = trunc iN %widesel to iK
5110 //
5111 // Note that C could be extended in any way because we don't care about
5112 // upper bits after truncation. It can't be abs pattern, because it would
5113 // look like:
5114 //
5115 // select i1 %cond, x, -x.
5116 //
5117 // So only min/max pattern could be matched. Such match requires widened C
5118 // == CmpConst. That is why set widened C = CmpConst, condition trunc
5119 // CmpConst == C is checked below.
5120 CastedTo = CmpConst;
5121 } else {
5122 CastedTo = ConstantExpr::getIntegerCast(C, SrcTy, CmpI->isSigned());
5123 }
Sanjay Patel14a4b812017-01-29 16:34:57 +00005124 break;
5125 case Instruction::FPTrunc:
5126 CastedTo = ConstantExpr::getFPExtend(C, SrcTy, true);
5127 break;
5128 case Instruction::FPExt:
5129 CastedTo = ConstantExpr::getFPTrunc(C, SrcTy, true);
5130 break;
5131 case Instruction::FPToUI:
5132 CastedTo = ConstantExpr::getUIToFP(C, SrcTy, true);
5133 break;
5134 case Instruction::FPToSI:
5135 CastedTo = ConstantExpr::getSIToFP(C, SrcTy, true);
5136 break;
5137 case Instruction::UIToFP:
5138 CastedTo = ConstantExpr::getFPToUI(C, SrcTy, true);
5139 break;
5140 case Instruction::SIToFP:
5141 CastedTo = ConstantExpr::getFPToSI(C, SrcTy, true);
5142 break;
5143 default:
5144 break;
5145 }
David Majnemerd2a074b2016-04-29 18:40:34 +00005146
5147 if (!CastedTo)
5148 return nullptr;
5149
David Majnemerd2a074b2016-04-29 18:40:34 +00005150 // Make sure the cast doesn't lose any information.
Sanjay Patel14a4b812017-01-29 16:34:57 +00005151 Constant *CastedBack =
5152 ConstantExpr::getCast(*CastOp, CastedTo, C->getType(), true);
David Majnemerd2a074b2016-04-29 18:40:34 +00005153 if (CastedBack != C)
5154 return nullptr;
5155
5156 return CastedTo;
James Molloy270ef8c2015-05-15 16:04:50 +00005157}
5158
Sanjay Patele8dc0902016-05-23 17:57:54 +00005159SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
Sanjay Patel1d91ec32018-01-24 15:20:37 +00005160 Instruction::CastOps *CastOp,
5161 unsigned Depth) {
5162 if (Depth >= MaxDepth)
5163 return {SPF_UNKNOWN, SPNB_NA, false};
5164
James Molloy270ef8c2015-05-15 16:04:50 +00005165 SelectInst *SI = dyn_cast<SelectInst>(V);
James Molloy134bec22015-08-11 09:12:57 +00005166 if (!SI) return {SPF_UNKNOWN, SPNB_NA, false};
James Molloy270ef8c2015-05-15 16:04:50 +00005167
James Molloy134bec22015-08-11 09:12:57 +00005168 CmpInst *CmpI = dyn_cast<CmpInst>(SI->getCondition());
5169 if (!CmpI) return {SPF_UNKNOWN, SPNB_NA, false};
James Molloy270ef8c2015-05-15 16:04:50 +00005170
Joseph Tremoulet3bc6e2a2019-06-13 15:24:11 +00005171 Value *TrueVal = SI->getTrueValue();
5172 Value *FalseVal = SI->getFalseValue();
5173
5174 return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS,
5175 CastOp, Depth);
5176}
5177
5178SelectPatternResult llvm::matchDecomposedSelectPattern(
5179 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
5180 Instruction::CastOps *CastOp, unsigned Depth) {
James Molloy134bec22015-08-11 09:12:57 +00005181 CmpInst::Predicate Pred = CmpI->getPredicate();
James Molloy270ef8c2015-05-15 16:04:50 +00005182 Value *CmpLHS = CmpI->getOperand(0);
5183 Value *CmpRHS = CmpI->getOperand(1);
James Molloy134bec22015-08-11 09:12:57 +00005184 FastMathFlags FMF;
5185 if (isa<FPMathOperator>(CmpI))
5186 FMF = CmpI->getFastMathFlags();
James Molloy270ef8c2015-05-15 16:04:50 +00005187
5188 // Bail out early.
5189 if (CmpI->isEquality())
James Molloy134bec22015-08-11 09:12:57 +00005190 return {SPF_UNKNOWN, SPNB_NA, false};
James Molloy270ef8c2015-05-15 16:04:50 +00005191
5192 // Deal with type mismatches.
5193 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
Sanjay Patel9a399792017-12-26 15:09:19 +00005194 if (Value *C = lookThroughCast(CmpI, TrueVal, FalseVal, CastOp)) {
5195 // If this is a potential fmin/fmax with a cast to integer, then ignore
5196 // -0.0 because there is no corresponding integer value.
5197 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
5198 FMF.setNoSignedZeros();
James Molloy134bec22015-08-11 09:12:57 +00005199 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
James Molloy270ef8c2015-05-15 16:04:50 +00005200 cast<CastInst>(TrueVal)->getOperand(0), C,
Sanjay Patel1d91ec32018-01-24 15:20:37 +00005201 LHS, RHS, Depth);
Sanjay Patel9a399792017-12-26 15:09:19 +00005202 }
5203 if (Value *C = lookThroughCast(CmpI, FalseVal, TrueVal, CastOp)) {
5204 // If this is a potential fmin/fmax with a cast to integer, then ignore
5205 // -0.0 because there is no corresponding integer value.
5206 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
5207 FMF.setNoSignedZeros();
James Molloy134bec22015-08-11 09:12:57 +00005208 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
James Molloy270ef8c2015-05-15 16:04:50 +00005209 C, cast<CastInst>(FalseVal)->getOperand(0),
Sanjay Patel1d91ec32018-01-24 15:20:37 +00005210 LHS, RHS, Depth);
Sanjay Patel9a399792017-12-26 15:09:19 +00005211 }
James Molloy270ef8c2015-05-15 16:04:50 +00005212 }
James Molloy134bec22015-08-11 09:12:57 +00005213 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal,
Sanjay Patel1d91ec32018-01-24 15:20:37 +00005214 LHS, RHS, Depth);
James Molloy270ef8c2015-05-15 16:04:50 +00005215}
Sanjoy Dasa7e13782015-10-24 05:37:35 +00005216
Sanjay Patel7ed0bc22018-03-06 16:57:55 +00005217CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
5218 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
5219 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
5220 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
5221 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
5222 if (SPF == SPF_FMINNUM)
5223 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
5224 if (SPF == SPF_FMAXNUM)
5225 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
5226 llvm_unreachable("unhandled!");
5227}
5228
5229SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
5230 if (SPF == SPF_SMIN) return SPF_SMAX;
5231 if (SPF == SPF_UMIN) return SPF_UMAX;
5232 if (SPF == SPF_SMAX) return SPF_SMIN;
5233 if (SPF == SPF_UMAX) return SPF_UMIN;
5234 llvm_unreachable("unhandled!");
5235}
5236
5237CmpInst::Predicate llvm::getInverseMinMaxPred(SelectPatternFlavor SPF) {
5238 return getMinMaxPred(getInverseMinMaxFlavor(SPF));
5239}
5240
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005241/// Return true if "icmp Pred LHS RHS" is always true.
Chad Rosiere42b44b2017-07-28 14:39:06 +00005242static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
5243 const Value *RHS, const DataLayout &DL,
5244 unsigned Depth) {
Sanjoy Dasaf1400f2015-11-10 23:56:15 +00005245 assert(!LHS->getType()->isVectorTy() && "TODO: extend to handle vectors!");
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005246 if (ICmpInst::isTrueWhenEqual(Pred) && LHS == RHS)
5247 return true;
5248
5249 switch (Pred) {
5250 default:
5251 return false;
5252
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005253 case CmpInst::ICMP_SLE: {
Sanjoy Dasaf1400f2015-11-10 23:56:15 +00005254 const APInt *C;
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005255
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005256 // LHS s<= LHS +_{nsw} C if C >= 0
Sanjoy Dasdc26df42015-11-11 00:16:41 +00005257 if (match(RHS, m_NSWAdd(m_Specific(LHS), m_APInt(C))))
Sanjoy Dasaf1400f2015-11-10 23:56:15 +00005258 return !C->isNegative();
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005259 return false;
5260 }
5261
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005262 case CmpInst::ICMP_ULE: {
Sanjoy Dasaf1400f2015-11-10 23:56:15 +00005263 const APInt *C;
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005264
Sanjoy Dasdc26df42015-11-11 00:16:41 +00005265 // LHS u<= LHS +_{nuw} C for any C
5266 if (match(RHS, m_NUWAdd(m_Specific(LHS), m_APInt(C))))
Sanjoy Dasc01b4d22015-11-06 19:01:03 +00005267 return true;
Sanjoy Das92568102015-11-10 23:56:20 +00005268
5269 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
Pete Cooper35b00d52016-08-13 01:05:32 +00005270 auto MatchNUWAddsToSameValue = [&](const Value *A, const Value *B,
5271 const Value *&X,
Sanjoy Das92568102015-11-10 23:56:20 +00005272 const APInt *&CA, const APInt *&CB) {
5273 if (match(A, m_NUWAdd(m_Value(X), m_APInt(CA))) &&
5274 match(B, m_NUWAdd(m_Specific(X), m_APInt(CB))))
5275 return true;
5276
5277 // If X & C == 0 then (X | C) == X +_{nuw} C
5278 if (match(A, m_Or(m_Value(X), m_APInt(CA))) &&
5279 match(B, m_Or(m_Specific(X), m_APInt(CB)))) {
Craig Topperb45eabc2017-04-26 16:39:58 +00005280 KnownBits Known(CA->getBitWidth());
Chad Rosiere42b44b2017-07-28 14:39:06 +00005281 computeKnownBits(X, Known, DL, Depth + 1, /*AC*/ nullptr,
5282 /*CxtI*/ nullptr, /*DT*/ nullptr);
Craig Topperb45eabc2017-04-26 16:39:58 +00005283 if (CA->isSubsetOf(Known.Zero) && CB->isSubsetOf(Known.Zero))
Sanjoy Das92568102015-11-10 23:56:20 +00005284 return true;
5285 }
5286
5287 return false;
5288 };
5289
Pete Cooper35b00d52016-08-13 01:05:32 +00005290 const Value *X;
Sanjoy Das92568102015-11-10 23:56:20 +00005291 const APInt *CLHS, *CRHS;
Sanjoy Dasdc26df42015-11-11 00:16:41 +00005292 if (MatchNUWAddsToSameValue(LHS, RHS, X, CLHS, CRHS))
5293 return CLHS->ule(*CRHS);
Sanjoy Das92568102015-11-10 23:56:20 +00005294
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005295 return false;
5296 }
5297 }
5298}
5299
5300/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
Chad Rosier41dd31f2016-04-20 19:15:26 +00005301/// ALHS ARHS" is true. Otherwise, return None.
5302static Optional<bool>
Pete Cooper35b00d52016-08-13 01:05:32 +00005303isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
Chad Rosiere42b44b2017-07-28 14:39:06 +00005304 const Value *ARHS, const Value *BLHS, const Value *BRHS,
5305 const DataLayout &DL, unsigned Depth) {
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005306 switch (Pred) {
5307 default:
Chad Rosier41dd31f2016-04-20 19:15:26 +00005308 return None;
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005309
5310 case CmpInst::ICMP_SLT:
5311 case CmpInst::ICMP_SLE:
Chad Rosiere42b44b2017-07-28 14:39:06 +00005312 if (isTruePredicate(CmpInst::ICMP_SLE, BLHS, ALHS, DL, Depth) &&
5313 isTruePredicate(CmpInst::ICMP_SLE, ARHS, BRHS, DL, Depth))
Chad Rosier41dd31f2016-04-20 19:15:26 +00005314 return true;
5315 return None;
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005316
5317 case CmpInst::ICMP_ULT:
5318 case CmpInst::ICMP_ULE:
Chad Rosiere42b44b2017-07-28 14:39:06 +00005319 if (isTruePredicate(CmpInst::ICMP_ULE, BLHS, ALHS, DL, Depth) &&
5320 isTruePredicate(CmpInst::ICMP_ULE, ARHS, BRHS, DL, Depth))
Chad Rosier41dd31f2016-04-20 19:15:26 +00005321 return true;
5322 return None;
Sanjoy Das9349dcc2015-11-06 19:00:57 +00005323 }
5324}
5325
Chad Rosier226a7342016-05-05 17:41:19 +00005326/// Return true if the operands of the two compares match. IsSwappedOps is true
5327/// when the operands match, but are swapped.
Pete Cooper35b00d52016-08-13 01:05:32 +00005328static bool isMatchingOps(const Value *ALHS, const Value *ARHS,
5329 const Value *BLHS, const Value *BRHS,
Chad Rosier226a7342016-05-05 17:41:19 +00005330 bool &IsSwappedOps) {
5331
5332 bool IsMatchingOps = (ALHS == BLHS && ARHS == BRHS);
5333 IsSwappedOps = (ALHS == BRHS && ARHS == BLHS);
5334 return IsMatchingOps || IsSwappedOps;
5335}
5336
Sanjay Patel798c5982018-12-19 16:49:18 +00005337/// Return true if "icmp1 APred X, Y" implies "icmp2 BPred X, Y" is true.
5338/// Return false if "icmp1 APred X, Y" implies "icmp2 BPred X, Y" is false.
5339/// Otherwise, return None if we can't infer anything.
Chad Rosier41dd31f2016-04-20 19:15:26 +00005340static Optional<bool> isImpliedCondMatchingOperands(CmpInst::Predicate APred,
Chad Rosier41dd31f2016-04-20 19:15:26 +00005341 CmpInst::Predicate BPred,
Sanjay Patel798c5982018-12-19 16:49:18 +00005342 bool AreSwappedOps) {
5343 // Canonicalize the predicate as if the operands were not commuted.
5344 if (AreSwappedOps)
Chad Rosierb7dfbb42016-04-19 17:19:14 +00005345 BPred = ICmpInst::getSwappedPredicate(BPred);
Sanjay Patel798c5982018-12-19 16:49:18 +00005346
Chad Rosier99bc4802016-04-21 16:18:02 +00005347 if (CmpInst::isImpliedTrueByMatchingCmp(APred, BPred))
Chad Rosierb7dfbb42016-04-19 17:19:14 +00005348 return true;
Chad Rosier99bc4802016-04-21 16:18:02 +00005349 if (CmpInst::isImpliedFalseByMatchingCmp(APred, BPred))
Chad Rosier41dd31f2016-04-20 19:15:26 +00005350 return false;
Chad Rosierb7dfbb42016-04-19 17:19:14 +00005351
Chad Rosier41dd31f2016-04-20 19:15:26 +00005352 return None;
Chad Rosierb7dfbb42016-04-19 17:19:14 +00005353}
5354
Sanjay Patel798c5982018-12-19 16:49:18 +00005355/// Return true if "icmp APred X, C1" implies "icmp BPred X, C2" is true.
5356/// Return false if "icmp APred X, C1" implies "icmp BPred X, C2" is false.
5357/// Otherwise, return None if we can't infer anything.
Chad Rosier25cfb7d2016-05-05 15:39:18 +00005358static Optional<bool>
Sanjay Patel798c5982018-12-19 16:49:18 +00005359isImpliedCondMatchingImmOperands(CmpInst::Predicate APred,
Pete Cooper35b00d52016-08-13 01:05:32 +00005360 const ConstantInt *C1,
5361 CmpInst::Predicate BPred,
Sanjay Patel798c5982018-12-19 16:49:18 +00005362 const ConstantInt *C2) {
Chad Rosier25cfb7d2016-05-05 15:39:18 +00005363 ConstantRange DomCR =
5364 ConstantRange::makeExactICmpRegion(APred, C1->getValue());
5365 ConstantRange CR =
5366 ConstantRange::makeAllowedICmpRegion(BPred, C2->getValue());
5367 ConstantRange Intersection = DomCR.intersectWith(CR);
5368 ConstantRange Difference = DomCR.difference(CR);
5369 if (Intersection.isEmptySet())
5370 return false;
5371 if (Difference.isEmptySet())
5372 return true;
5373 return None;
5374}
5375
Chad Rosier2f498032017-07-28 18:47:43 +00005376/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
5377/// false. Otherwise, return None if we can't infer anything.
5378static Optional<bool> isImpliedCondICmps(const ICmpInst *LHS,
5379 const ICmpInst *RHS,
Chad Rosierdfd1de62017-08-01 20:18:54 +00005380 const DataLayout &DL, bool LHSIsTrue,
Chad Rosier2f498032017-07-28 18:47:43 +00005381 unsigned Depth) {
5382 Value *ALHS = LHS->getOperand(0);
5383 Value *ARHS = LHS->getOperand(1);
Chad Rosiera72a9ff2017-07-06 20:00:25 +00005384 // The rest of the logic assumes the LHS condition is true. If that's not the
5385 // case, invert the predicate to make it so.
Chad Rosier2f498032017-07-28 18:47:43 +00005386 ICmpInst::Predicate APred =
Chad Rosierdfd1de62017-08-01 20:18:54 +00005387 LHSIsTrue ? LHS->getPredicate() : LHS->getInversePredicate();
Chad Rosier2f498032017-07-28 18:47:43 +00005388
5389 Value *BLHS = RHS->getOperand(0);
5390 Value *BRHS = RHS->getOperand(1);
5391 ICmpInst::Predicate BPred = RHS->getPredicate();
Chad Rosiere2cbd132016-04-25 17:23:36 +00005392
Chad Rosier226a7342016-05-05 17:41:19 +00005393 // Can we infer anything when the two compares have matching operands?
Sanjay Patel798c5982018-12-19 16:49:18 +00005394 bool AreSwappedOps;
5395 if (isMatchingOps(ALHS, ARHS, BLHS, BRHS, AreSwappedOps)) {
Chad Rosier226a7342016-05-05 17:41:19 +00005396 if (Optional<bool> Implication = isImpliedCondMatchingOperands(
Sanjay Patel798c5982018-12-19 16:49:18 +00005397 APred, BPred, AreSwappedOps))
Chad Rosier25cfb7d2016-05-05 15:39:18 +00005398 return Implication;
Chad Rosier226a7342016-05-05 17:41:19 +00005399 // No amount of additional analysis will infer the second condition, so
5400 // early exit.
5401 return None;
5402 }
5403
5404 // Can we infer anything when the LHS operands match and the RHS operands are
5405 // constants (not necessarily matching)?
5406 if (ALHS == BLHS && isa<ConstantInt>(ARHS) && isa<ConstantInt>(BRHS)) {
5407 if (Optional<bool> Implication = isImpliedCondMatchingImmOperands(
Sanjay Patel798c5982018-12-19 16:49:18 +00005408 APred, cast<ConstantInt>(ARHS), BPred, cast<ConstantInt>(BRHS)))
Chad Rosier226a7342016-05-05 17:41:19 +00005409 return Implication;
5410 // No amount of additional analysis will infer the second condition, so
5411 // early exit.
5412 return None;
Chad Rosier25cfb7d2016-05-05 15:39:18 +00005413 }
5414
Chad Rosier41dd31f2016-04-20 19:15:26 +00005415 if (APred == BPred)
Chad Rosiere42b44b2017-07-28 14:39:06 +00005416 return isImpliedCondOperands(APred, ALHS, ARHS, BLHS, BRHS, DL, Depth);
Chad Rosier41dd31f2016-04-20 19:15:26 +00005417 return None;
Sanjoy Das3ef1e682015-10-28 03:20:19 +00005418}
Chad Rosier2f498032017-07-28 18:47:43 +00005419
Chad Rosierf73a10d2017-08-01 19:22:36 +00005420/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
5421/// false. Otherwise, return None if we can't infer anything. We expect the
5422/// RHS to be an icmp and the LHS to be an 'and' or an 'or' instruction.
5423static Optional<bool> isImpliedCondAndOr(const BinaryOperator *LHS,
5424 const ICmpInst *RHS,
Chad Rosierdfd1de62017-08-01 20:18:54 +00005425 const DataLayout &DL, bool LHSIsTrue,
Chad Rosierf73a10d2017-08-01 19:22:36 +00005426 unsigned Depth) {
5427 // The LHS must be an 'or' or an 'and' instruction.
5428 assert((LHS->getOpcode() == Instruction::And ||
5429 LHS->getOpcode() == Instruction::Or) &&
5430 "Expected LHS to be 'and' or 'or'.");
5431
Davide Italiano1a943a92017-08-09 16:06:54 +00005432 assert(Depth <= MaxDepth && "Hit recursion limit");
Chad Rosierf73a10d2017-08-01 19:22:36 +00005433
5434 // If the result of an 'or' is false, then we know both legs of the 'or' are
5435 // false. Similarly, if the result of an 'and' is true, then we know both
5436 // legs of the 'and' are true.
5437 Value *ALHS, *ARHS;
Chad Rosierdfd1de62017-08-01 20:18:54 +00005438 if ((!LHSIsTrue && match(LHS, m_Or(m_Value(ALHS), m_Value(ARHS)))) ||
5439 (LHSIsTrue && match(LHS, m_And(m_Value(ALHS), m_Value(ARHS))))) {
Chad Rosierf73a10d2017-08-01 19:22:36 +00005440 // FIXME: Make this non-recursion.
5441 if (Optional<bool> Implication =
Chad Rosierdfd1de62017-08-01 20:18:54 +00005442 isImpliedCondition(ALHS, RHS, DL, LHSIsTrue, Depth + 1))
Chad Rosierf73a10d2017-08-01 19:22:36 +00005443 return Implication;
5444 if (Optional<bool> Implication =
Chad Rosierdfd1de62017-08-01 20:18:54 +00005445 isImpliedCondition(ARHS, RHS, DL, LHSIsTrue, Depth + 1))
Chad Rosierf73a10d2017-08-01 19:22:36 +00005446 return Implication;
5447 return None;
5448 }
5449 return None;
5450}
5451
Chad Rosier2f498032017-07-28 18:47:43 +00005452Optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
Chad Rosierdfd1de62017-08-01 20:18:54 +00005453 const DataLayout &DL, bool LHSIsTrue,
Chad Rosier2f498032017-07-28 18:47:43 +00005454 unsigned Depth) {
Davide Italiano30e51942017-08-09 15:13:50 +00005455 // Bail out when we hit the limit.
5456 if (Depth == MaxDepth)
5457 return None;
5458
Chad Rosierf73a10d2017-08-01 19:22:36 +00005459 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
5460 // example.
Chad Rosier2f498032017-07-28 18:47:43 +00005461 if (LHS->getType() != RHS->getType())
5462 return None;
5463
5464 Type *OpTy = LHS->getType();
Chad Rosierf73a10d2017-08-01 19:22:36 +00005465 assert(OpTy->isIntOrIntVectorTy(1) && "Expected integer type only!");
Chad Rosier2f498032017-07-28 18:47:43 +00005466
5467 // LHS ==> RHS by definition
5468 if (LHS == RHS)
Chad Rosierdfd1de62017-08-01 20:18:54 +00005469 return LHSIsTrue;
Chad Rosier2f498032017-07-28 18:47:43 +00005470
Chad Rosierf73a10d2017-08-01 19:22:36 +00005471 // FIXME: Extending the code below to handle vectors.
Chad Rosier2f498032017-07-28 18:47:43 +00005472 if (OpTy->isVectorTy())
Chad Rosier2f498032017-07-28 18:47:43 +00005473 return None;
Chad Rosierf73a10d2017-08-01 19:22:36 +00005474
Chad Rosier2f498032017-07-28 18:47:43 +00005475 assert(OpTy->isIntegerTy(1) && "implied by above");
5476
Chad Rosier2f498032017-07-28 18:47:43 +00005477 // Both LHS and RHS are icmps.
Chad Rosierf73a10d2017-08-01 19:22:36 +00005478 const ICmpInst *LHSCmp = dyn_cast<ICmpInst>(LHS);
5479 const ICmpInst *RHSCmp = dyn_cast<ICmpInst>(RHS);
5480 if (LHSCmp && RHSCmp)
Chad Rosierdfd1de62017-08-01 20:18:54 +00005481 return isImpliedCondICmps(LHSCmp, RHSCmp, DL, LHSIsTrue, Depth);
Chad Rosier2f498032017-07-28 18:47:43 +00005482
Chad Rosierf73a10d2017-08-01 19:22:36 +00005483 // The LHS should be an 'or' or an 'and' instruction. We expect the RHS to be
5484 // an icmp. FIXME: Add support for and/or on the RHS.
5485 const BinaryOperator *LHSBO = dyn_cast<BinaryOperator>(LHS);
5486 if (LHSBO && RHSCmp) {
5487 if ((LHSBO->getOpcode() == Instruction::And ||
5488 LHSBO->getOpcode() == Instruction::Or))
Chad Rosierdfd1de62017-08-01 20:18:54 +00005489 return isImpliedCondAndOr(LHSBO, RHSCmp, DL, LHSIsTrue, Depth);
Chad Rosier2f498032017-07-28 18:47:43 +00005490 }
Chad Rosierf73a10d2017-08-01 19:22:36 +00005491 return None;
Chad Rosier2f498032017-07-28 18:47:43 +00005492}
Sanjay Patel7d82d372018-12-02 13:26:03 +00005493
5494Optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
5495 const Instruction *ContextI,
5496 const DataLayout &DL) {
5497 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
5498 if (!ContextI || !ContextI->getParent())
5499 return None;
5500
5501 // TODO: This is a poor/cheap way to determine dominance. Should we use a
5502 // dominator tree (eg, from a SimplifyQuery) instead?
5503 const BasicBlock *ContextBB = ContextI->getParent();
5504 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
5505 if (!PredBB)
5506 return None;
5507
5508 // We need a conditional branch in the predecessor.
5509 Value *PredCond;
5510 BasicBlock *TrueBB, *FalseBB;
5511 if (!match(PredBB->getTerminator(), m_Br(m_Value(PredCond), TrueBB, FalseBB)))
5512 return None;
5513
5514 // The branch should get simplified. Don't bother simplifying this condition.
5515 if (TrueBB == FalseBB)
5516 return None;
5517
5518 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
5519 "Predecessor block does not point to successor?");
5520
5521 // Is this condition implied by the predecessor condition?
5522 bool CondIsTrue = TrueBB == ContextBB;
5523 return isImpliedCondition(PredCond, Cond, DL, CondIsTrue);
5524}
Nikita Popov49097592019-03-09 21:17:42 +00005525
5526static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
5527 APInt &Upper, const InstrInfoQuery &IIQ) {
5528 unsigned Width = Lower.getBitWidth();
5529 const APInt *C;
5530 switch (BO.getOpcode()) {
5531 case Instruction::Add:
5532 if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) {
5533 // FIXME: If we have both nuw and nsw, we should reduce the range further.
5534 if (IIQ.hasNoUnsignedWrap(cast<OverflowingBinaryOperator>(&BO))) {
5535 // 'add nuw x, C' produces [C, UINT_MAX].
5536 Lower = *C;
5537 } else if (IIQ.hasNoSignedWrap(cast<OverflowingBinaryOperator>(&BO))) {
5538 if (C->isNegative()) {
5539 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
5540 Lower = APInt::getSignedMinValue(Width);
5541 Upper = APInt::getSignedMaxValue(Width) + *C + 1;
5542 } else {
5543 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
5544 Lower = APInt::getSignedMinValue(Width) + *C;
5545 Upper = APInt::getSignedMaxValue(Width) + 1;
5546 }
5547 }
5548 }
5549 break;
5550
5551 case Instruction::And:
5552 if (match(BO.getOperand(1), m_APInt(C)))
5553 // 'and x, C' produces [0, C].
5554 Upper = *C + 1;
5555 break;
5556
5557 case Instruction::Or:
5558 if (match(BO.getOperand(1), m_APInt(C)))
5559 // 'or x, C' produces [C, UINT_MAX].
5560 Lower = *C;
5561 break;
5562
5563 case Instruction::AShr:
5564 if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) {
5565 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
5566 Lower = APInt::getSignedMinValue(Width).ashr(*C);
5567 Upper = APInt::getSignedMaxValue(Width).ashr(*C) + 1;
5568 } else if (match(BO.getOperand(0), m_APInt(C))) {
5569 unsigned ShiftAmount = Width - 1;
5570 if (!C->isNullValue() && IIQ.isExact(&BO))
5571 ShiftAmount = C->countTrailingZeros();
5572 if (C->isNegative()) {
5573 // 'ashr C, x' produces [C, C >> (Width-1)]
5574 Lower = *C;
5575 Upper = C->ashr(ShiftAmount) + 1;
5576 } else {
5577 // 'ashr C, x' produces [C >> (Width-1), C]
5578 Lower = C->ashr(ShiftAmount);
5579 Upper = *C + 1;
5580 }
5581 }
5582 break;
5583
5584 case Instruction::LShr:
5585 if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) {
5586 // 'lshr x, C' produces [0, UINT_MAX >> C].
5587 Upper = APInt::getAllOnesValue(Width).lshr(*C) + 1;
5588 } else if (match(BO.getOperand(0), m_APInt(C))) {
5589 // 'lshr C, x' produces [C >> (Width-1), C].
5590 unsigned ShiftAmount = Width - 1;
5591 if (!C->isNullValue() && IIQ.isExact(&BO))
5592 ShiftAmount = C->countTrailingZeros();
5593 Lower = C->lshr(ShiftAmount);
5594 Upper = *C + 1;
5595 }
5596 break;
5597
5598 case Instruction::Shl:
5599 if (match(BO.getOperand(0), m_APInt(C))) {
5600 if (IIQ.hasNoUnsignedWrap(&BO)) {
5601 // 'shl nuw C, x' produces [C, C << CLZ(C)]
5602 Lower = *C;
5603 Upper = Lower.shl(Lower.countLeadingZeros()) + 1;
5604 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
5605 if (C->isNegative()) {
5606 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
5607 unsigned ShiftAmount = C->countLeadingOnes() - 1;
5608 Lower = C->shl(ShiftAmount);
5609 Upper = *C + 1;
5610 } else {
5611 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
5612 unsigned ShiftAmount = C->countLeadingZeros() - 1;
5613 Lower = *C;
5614 Upper = C->shl(ShiftAmount) + 1;
5615 }
5616 }
5617 }
5618 break;
5619
5620 case Instruction::SDiv:
5621 if (match(BO.getOperand(1), m_APInt(C))) {
5622 APInt IntMin = APInt::getSignedMinValue(Width);
5623 APInt IntMax = APInt::getSignedMaxValue(Width);
5624 if (C->isAllOnesValue()) {
5625 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
5626 // where C != -1 and C != 0 and C != 1
5627 Lower = IntMin + 1;
5628 Upper = IntMax + 1;
5629 } else if (C->countLeadingZeros() < Width - 1) {
5630 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
5631 // where C != -1 and C != 0 and C != 1
5632 Lower = IntMin.sdiv(*C);
5633 Upper = IntMax.sdiv(*C);
5634 if (Lower.sgt(Upper))
5635 std::swap(Lower, Upper);
5636 Upper = Upper + 1;
5637 assert(Upper != Lower && "Upper part of range has wrapped!");
5638 }
5639 } else if (match(BO.getOperand(0), m_APInt(C))) {
5640 if (C->isMinSignedValue()) {
5641 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
5642 Lower = *C;
5643 Upper = Lower.lshr(1) + 1;
5644 } else {
5645 // 'sdiv C, x' produces [-|C|, |C|].
5646 Upper = C->abs() + 1;
5647 Lower = (-Upper) + 1;
5648 }
5649 }
5650 break;
5651
5652 case Instruction::UDiv:
5653 if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) {
5654 // 'udiv x, C' produces [0, UINT_MAX / C].
5655 Upper = APInt::getMaxValue(Width).udiv(*C) + 1;
5656 } else if (match(BO.getOperand(0), m_APInt(C))) {
5657 // 'udiv C, x' produces [0, C].
5658 Upper = *C + 1;
5659 }
5660 break;
5661
5662 case Instruction::SRem:
5663 if (match(BO.getOperand(1), m_APInt(C))) {
5664 // 'srem x, C' produces (-|C|, |C|).
5665 Upper = C->abs();
5666 Lower = (-Upper) + 1;
5667 }
5668 break;
5669
5670 case Instruction::URem:
5671 if (match(BO.getOperand(1), m_APInt(C)))
5672 // 'urem x, C' produces [0, C).
5673 Upper = *C;
5674 break;
5675
5676 default:
5677 break;
5678 }
5679}
5680
5681static void setLimitsForIntrinsic(const IntrinsicInst &II, APInt &Lower,
5682 APInt &Upper) {
5683 unsigned Width = Lower.getBitWidth();
5684 const APInt *C;
5685 switch (II.getIntrinsicID()) {
5686 case Intrinsic::uadd_sat:
5687 // uadd.sat(x, C) produces [C, UINT_MAX].
5688 if (match(II.getOperand(0), m_APInt(C)) ||
5689 match(II.getOperand(1), m_APInt(C)))
5690 Lower = *C;
5691 break;
5692 case Intrinsic::sadd_sat:
5693 if (match(II.getOperand(0), m_APInt(C)) ||
5694 match(II.getOperand(1), m_APInt(C))) {
5695 if (C->isNegative()) {
5696 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
5697 Lower = APInt::getSignedMinValue(Width);
5698 Upper = APInt::getSignedMaxValue(Width) + *C + 1;
5699 } else {
5700 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
5701 Lower = APInt::getSignedMinValue(Width) + *C;
5702 Upper = APInt::getSignedMaxValue(Width) + 1;
5703 }
5704 }
5705 break;
5706 case Intrinsic::usub_sat:
5707 // usub.sat(C, x) produces [0, C].
5708 if (match(II.getOperand(0), m_APInt(C)))
5709 Upper = *C + 1;
5710 // usub.sat(x, C) produces [0, UINT_MAX - C].
5711 else if (match(II.getOperand(1), m_APInt(C)))
5712 Upper = APInt::getMaxValue(Width) - *C + 1;
5713 break;
5714 case Intrinsic::ssub_sat:
5715 if (match(II.getOperand(0), m_APInt(C))) {
5716 if (C->isNegative()) {
5717 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
5718 Lower = APInt::getSignedMinValue(Width);
5719 Upper = *C - APInt::getSignedMinValue(Width) + 1;
5720 } else {
5721 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
5722 Lower = *C - APInt::getSignedMaxValue(Width);
5723 Upper = APInt::getSignedMaxValue(Width) + 1;
5724 }
5725 } else if (match(II.getOperand(1), m_APInt(C))) {
5726 if (C->isNegative()) {
5727 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
5728 Lower = APInt::getSignedMinValue(Width) - *C;
5729 Upper = APInt::getSignedMaxValue(Width) + 1;
5730 } else {
5731 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
5732 Lower = APInt::getSignedMinValue(Width);
5733 Upper = APInt::getSignedMaxValue(Width) - *C + 1;
5734 }
5735 }
5736 break;
5737 default:
5738 break;
5739 }
5740}
5741
Nikita Popovf89343b2019-03-18 21:20:03 +00005742static void setLimitsForSelectPattern(const SelectInst &SI, APInt &Lower,
Craig Topper66c08432019-08-07 18:28:16 +00005743 APInt &Upper, const InstrInfoQuery &IIQ) {
Simon Pilgrimf62293e2019-09-23 13:15:52 +00005744 const Value *LHS = nullptr, *RHS = nullptr;
Nikita Popovf89343b2019-03-18 21:20:03 +00005745 SelectPatternResult R = matchSelectPattern(&SI, LHS, RHS);
5746 if (R.Flavor == SPF_UNKNOWN)
5747 return;
5748
5749 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
5750
Nikita Popov00b5eca2019-03-20 18:16:02 +00005751 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
5752 // If the negation part of the abs (in RHS) has the NSW flag,
5753 // then the result of abs(X) is [0..SIGNED_MAX],
5754 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
Nikita Popovf89343b2019-03-18 21:20:03 +00005755 Lower = APInt::getNullValue(BitWidth);
Craig Topper66c08432019-08-07 18:28:16 +00005756 if (match(RHS, m_Neg(m_Specific(LHS))) &&
5757 IIQ.hasNoSignedWrap(cast<Instruction>(RHS)))
Nikita Popov00b5eca2019-03-20 18:16:02 +00005758 Upper = APInt::getSignedMaxValue(BitWidth) + 1;
5759 else
5760 Upper = APInt::getSignedMinValue(BitWidth) + 1;
Nikita Popovf89343b2019-03-18 21:20:03 +00005761 return;
5762 }
5763
5764 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
5765 // The result of -abs(X) is <= 0.
5766 Lower = APInt::getSignedMinValue(BitWidth);
5767 Upper = APInt(BitWidth, 1);
5768 return;
5769 }
5770
Nikita Popov3db93ac2019-04-07 17:22:16 +00005771 const APInt *C;
5772 if (!match(LHS, m_APInt(C)) && !match(RHS, m_APInt(C)))
5773 return;
5774
5775 switch (R.Flavor) {
5776 case SPF_UMIN:
5777 Upper = *C + 1;
5778 break;
5779 case SPF_UMAX:
5780 Lower = *C;
5781 break;
5782 case SPF_SMIN:
5783 Lower = APInt::getSignedMinValue(BitWidth);
5784 Upper = *C + 1;
5785 break;
5786 case SPF_SMAX:
5787 Lower = *C;
5788 Upper = APInt::getSignedMaxValue(BitWidth) + 1;
5789 break;
5790 default:
5791 break;
5792 }
Nikita Popovf89343b2019-03-18 21:20:03 +00005793}
5794
Nikita Popov49097592019-03-09 21:17:42 +00005795ConstantRange llvm::computeConstantRange(const Value *V, bool UseInstrInfo) {
5796 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
5797
Nikita Popov20838192019-03-19 17:53:56 +00005798 const APInt *C;
5799 if (match(V, m_APInt(C)))
5800 return ConstantRange(*C);
5801
Nikita Popov49097592019-03-09 21:17:42 +00005802 InstrInfoQuery IIQ(UseInstrInfo);
5803 unsigned BitWidth = V->getType()->getScalarSizeInBits();
5804 APInt Lower = APInt(BitWidth, 0);
5805 APInt Upper = APInt(BitWidth, 0);
5806 if (auto *BO = dyn_cast<BinaryOperator>(V))
5807 setLimitsForBinOp(*BO, Lower, Upper, IIQ);
5808 else if (auto *II = dyn_cast<IntrinsicInst>(V))
5809 setLimitsForIntrinsic(*II, Lower, Upper);
Nikita Popovf89343b2019-03-18 21:20:03 +00005810 else if (auto *SI = dyn_cast<SelectInst>(V))
Craig Topper66c08432019-08-07 18:28:16 +00005811 setLimitsForSelectPattern(*SI, Lower, Upper, IIQ);
Nikita Popov49097592019-03-09 21:17:42 +00005812
Nikita Popovdbc3fba2019-04-21 15:22:54 +00005813 ConstantRange CR = ConstantRange::getNonEmpty(Lower, Upper);
Nikita Popov49097592019-03-09 21:17:42 +00005814
5815 if (auto *I = dyn_cast<Instruction>(V))
5816 if (auto *Range = IIQ.getMetadata(I, LLVMContext::MD_range))
5817 CR = CR.intersectWith(getConstantRangeFromMetadata(*Range));
5818
5819 return CR;
5820}
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005821
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005822static Optional<int64_t>
5823getOffsetFromIndex(const GEPOperator *GEP, unsigned Idx, const DataLayout &DL) {
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005824 // Skip over the first indices.
5825 gep_type_iterator GTI = gep_type_begin(GEP);
5826 for (unsigned i = 1; i != Idx; ++i, ++GTI)
5827 /*skip along*/;
5828
5829 // Compute the offset implied by the rest of the indices.
5830 int64_t Offset = 0;
5831 for (unsigned i = Idx, e = GEP->getNumOperands(); i != e; ++i, ++GTI) {
5832 ConstantInt *OpC = dyn_cast<ConstantInt>(GEP->getOperand(i));
5833 if (!OpC)
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005834 return None;
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005835 if (OpC->isZero())
5836 continue; // No offset.
5837
5838 // Handle struct indices, which add their field offset to the pointer.
5839 if (StructType *STy = GTI.getStructTypeOrNull()) {
5840 Offset += DL.getStructLayout(STy)->getElementOffset(OpC->getZExtValue());
5841 continue;
5842 }
5843
5844 // Otherwise, we have a sequential type like an array or vector. Multiply
5845 // the index by the ElementSize.
5846 uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType());
5847 Offset += Size * OpC->getSExtValue();
5848 }
5849
5850 return Offset;
5851}
5852
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005853Optional<int64_t> llvm::isPointerOffset(const Value *Ptr1, const Value *Ptr2,
5854 const DataLayout &DL) {
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005855 Ptr1 = Ptr1->stripPointerCasts();
5856 Ptr2 = Ptr2->stripPointerCasts();
5857
5858 // Handle the trivial case first.
5859 if (Ptr1 == Ptr2) {
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005860 return 0;
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005861 }
5862
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005863 const GEPOperator *GEP1 = dyn_cast<GEPOperator>(Ptr1);
5864 const GEPOperator *GEP2 = dyn_cast<GEPOperator>(Ptr2);
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005865
Rong Xu686fa4b2019-10-10 21:30:43 +00005866 // If one pointer is a GEP see if the GEP is a constant offset from the base,
5867 // as in "P" and "gep P, 1".
5868 // Also do this iteratively to handle the the following case:
5869 // Ptr_t1 = GEP Ptr1, c1
5870 // Ptr_t2 = GEP Ptr_t1, c2
5871 // Ptr2 = GEP Ptr_t2, c3
5872 // where we will return c1+c2+c3.
5873 // TODO: Handle the case when both Ptr1 and Ptr2 are GEPs of some common base
5874 // -- replace getOffsetFromBase with getOffsetAndBase, check that the bases
5875 // are the same, and return the difference between offsets.
5876 auto getOffsetFromBase = [&DL](const GEPOperator *GEP,
5877 const Value *Ptr) -> Optional<int64_t> {
5878 const GEPOperator *GEP_T = GEP;
5879 int64_t OffsetVal = 0;
5880 bool HasSameBase = false;
5881 while (GEP_T) {
5882 auto Offset = getOffsetFromIndex(GEP_T, 1, DL);
5883 if (!Offset)
5884 return None;
5885 OffsetVal += *Offset;
5886 auto Op0 = GEP_T->getOperand(0)->stripPointerCasts();
5887 if (Op0 == Ptr) {
5888 HasSameBase = true;
5889 break;
5890 }
5891 GEP_T = dyn_cast<GEPOperator>(Op0);
5892 }
5893 if (!HasSameBase)
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005894 return None;
Rong Xu686fa4b2019-10-10 21:30:43 +00005895 return OffsetVal;
5896 };
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005897
Rong Xu686fa4b2019-10-10 21:30:43 +00005898 if (GEP1) {
5899 auto Offset = getOffsetFromBase(GEP1, Ptr2);
5900 if (Offset)
5901 return -*Offset;
5902 }
5903 if (GEP2) {
5904 auto Offset = getOffsetFromBase(GEP2, Ptr1);
5905 if (Offset)
5906 return Offset;
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005907 }
5908
5909 // Right now we handle the case when Ptr1/Ptr2 are both GEPs with an identical
5910 // base. After that base, they may have some number of common (and
5911 // potentially variable) indices. After that they handle some constant
5912 // offset, which determines their offset from each other. At this point, we
5913 // handle no other case.
5914 if (!GEP1 || !GEP2 || GEP1->getOperand(0) != GEP2->getOperand(0))
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005915 return None;
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005916
5917 // Skip any common indices and track the GEP types.
5918 unsigned Idx = 1;
5919 for (; Idx != GEP1->getNumOperands() && Idx != GEP2->getNumOperands(); ++Idx)
5920 if (GEP1->getOperand(Idx) != GEP2->getOperand(Idx))
5921 break;
5922
Evgeniy Stepanov55ccd162019-08-19 21:08:04 +00005923 auto Offset1 = getOffsetFromIndex(GEP1, Idx, DL);
5924 auto Offset2 = getOffsetFromIndex(GEP2, Idx, DL);
5925 if (!Offset1 || !Offset2)
5926 return None;
5927 return *Offset2 - *Offset1;
Evgeniy Stepanov75344952019-08-15 22:58:28 +00005928}