Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1 | //===- InstCombineMulDivRem.cpp -------------------------------------------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file implements the visit functions for mul, fmul, sdiv, udiv, fdiv, |
| 11 | // srem, urem, frem. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "InstCombine.h" |
Duncan Sands | 82fdab3 | 2010-12-21 14:00:22 +0000 | [diff] [blame] | 16 | #include "llvm/Analysis/InstructionSimplify.h" |
Chandler Carruth | 0b8c9a8 | 2013-01-02 11:36:10 +0000 | [diff] [blame] | 17 | #include "llvm/IR/IntrinsicInst.h" |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 18 | #include "llvm/Support/PatternMatch.h" |
| 19 | using namespace llvm; |
| 20 | using namespace PatternMatch; |
| 21 | |
Chris Lattner | 1add46d | 2011-05-22 18:18:41 +0000 | [diff] [blame] | 22 | |
| 23 | /// simplifyValueKnownNonZero - The specific integer value is used in a context |
| 24 | /// where it is known to be non-zero. If this allows us to simplify the |
| 25 | /// computation, do so and return the new operand, otherwise return null. |
| 26 | static Value *simplifyValueKnownNonZero(Value *V, InstCombiner &IC) { |
| 27 | // If V has multiple uses, then we would have to do more analysis to determine |
| 28 | // if this is safe. For example, the use could be in dynamically unreached |
| 29 | // code. |
| 30 | if (!V->hasOneUse()) return 0; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 31 | |
Chris Lattner | 613f1a3 | 2011-05-23 00:32:19 +0000 | [diff] [blame] | 32 | bool MadeChange = false; |
| 33 | |
Chris Lattner | 1add46d | 2011-05-22 18:18:41 +0000 | [diff] [blame] | 34 | // ((1 << A) >>u B) --> (1 << (A-B)) |
| 35 | // Because V cannot be zero, we know that B is less than A. |
Chris Lattner | 6083bb9 | 2011-05-23 00:09:55 +0000 | [diff] [blame] | 36 | Value *A = 0, *B = 0, *PowerOf2 = 0; |
| 37 | if (match(V, m_LShr(m_OneUse(m_Shl(m_Value(PowerOf2), m_Value(A))), |
Chris Lattner | 1add46d | 2011-05-22 18:18:41 +0000 | [diff] [blame] | 38 | m_Value(B))) && |
| 39 | // The "1" can be any value known to be a power of 2. |
Rafael Espindola | dbaa237 | 2012-12-13 03:37:24 +0000 | [diff] [blame] | 40 | isKnownToBeAPowerOfTwo(PowerOf2)) { |
Benjamin Kramer | a9390a4 | 2011-09-27 20:39:19 +0000 | [diff] [blame] | 41 | A = IC.Builder->CreateSub(A, B); |
Chris Lattner | 6083bb9 | 2011-05-23 00:09:55 +0000 | [diff] [blame] | 42 | return IC.Builder->CreateShl(PowerOf2, A); |
Chris Lattner | 1add46d | 2011-05-22 18:18:41 +0000 | [diff] [blame] | 43 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 44 | |
Chris Lattner | 613f1a3 | 2011-05-23 00:32:19 +0000 | [diff] [blame] | 45 | // (PowerOfTwo >>u B) --> isExact since shifting out the result would make it |
| 46 | // inexact. Similarly for <<. |
| 47 | if (BinaryOperator *I = dyn_cast<BinaryOperator>(V)) |
Rafael Espindola | dbaa237 | 2012-12-13 03:37:24 +0000 | [diff] [blame] | 48 | if (I->isLogicalShift() && isKnownToBeAPowerOfTwo(I->getOperand(0))) { |
Chris Lattner | 613f1a3 | 2011-05-23 00:32:19 +0000 | [diff] [blame] | 49 | // We know that this is an exact/nuw shift and that the input is a |
| 50 | // non-zero context as well. |
| 51 | if (Value *V2 = simplifyValueKnownNonZero(I->getOperand(0), IC)) { |
| 52 | I->setOperand(0, V2); |
| 53 | MadeChange = true; |
| 54 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 55 | |
Chris Lattner | 613f1a3 | 2011-05-23 00:32:19 +0000 | [diff] [blame] | 56 | if (I->getOpcode() == Instruction::LShr && !I->isExact()) { |
| 57 | I->setIsExact(); |
| 58 | MadeChange = true; |
| 59 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 60 | |
Chris Lattner | 613f1a3 | 2011-05-23 00:32:19 +0000 | [diff] [blame] | 61 | if (I->getOpcode() == Instruction::Shl && !I->hasNoUnsignedWrap()) { |
| 62 | I->setHasNoUnsignedWrap(); |
| 63 | MadeChange = true; |
| 64 | } |
| 65 | } |
| 66 | |
Chris Lattner | 6c9b8d3 | 2011-05-22 18:26:48 +0000 | [diff] [blame] | 67 | // TODO: Lots more we could do here: |
Chris Lattner | 6c9b8d3 | 2011-05-22 18:26:48 +0000 | [diff] [blame] | 68 | // If V is a phi node, we can call this on each of its operands. |
| 69 | // "select cond, X, 0" can simplify to "X". |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 70 | |
Chris Lattner | 613f1a3 | 2011-05-23 00:32:19 +0000 | [diff] [blame] | 71 | return MadeChange ? V : 0; |
Chris Lattner | 1add46d | 2011-05-22 18:18:41 +0000 | [diff] [blame] | 72 | } |
| 73 | |
| 74 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 75 | /// MultiplyOverflows - True if the multiply can not be expressed in an int |
| 76 | /// this size. |
| 77 | static bool MultiplyOverflows(ConstantInt *C1, ConstantInt *C2, bool sign) { |
| 78 | uint32_t W = C1->getBitWidth(); |
| 79 | APInt LHSExt = C1->getValue(), RHSExt = C2->getValue(); |
| 80 | if (sign) { |
Jay Foad | 40f8f62 | 2010-12-07 08:25:19 +0000 | [diff] [blame] | 81 | LHSExt = LHSExt.sext(W * 2); |
| 82 | RHSExt = RHSExt.sext(W * 2); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 83 | } else { |
Jay Foad | 40f8f62 | 2010-12-07 08:25:19 +0000 | [diff] [blame] | 84 | LHSExt = LHSExt.zext(W * 2); |
| 85 | RHSExt = RHSExt.zext(W * 2); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 86 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 87 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 88 | APInt MulExt = LHSExt * RHSExt; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 89 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 90 | if (!sign) |
| 91 | return MulExt.ugt(APInt::getLowBitsSet(W * 2, W)); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 92 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 93 | APInt Min = APInt::getSignedMinValue(W).sext(W * 2); |
| 94 | APInt Max = APInt::getSignedMaxValue(W).sext(W * 2); |
| 95 | return MulExt.slt(Min) || MulExt.sgt(Max); |
| 96 | } |
| 97 | |
Rafael Espindola | 4f3d7ee | 2013-05-31 14:27:15 +0000 | [diff] [blame] | 98 | /// \brief A helper routine of InstCombiner::visitMul(). |
| 99 | /// |
| 100 | /// If C is a vector of known powers of 2, then this function returns |
| 101 | /// a new vector obtained from C replacing each element with its logBase2. |
| 102 | /// Return a null pointer otherwise. |
| 103 | static Constant *getLogBase2Vector(ConstantDataVector *CV) { |
| 104 | const APInt *IVal; |
| 105 | SmallVector<Constant *, 4> Elts; |
| 106 | |
| 107 | for (unsigned I = 0, E = CV->getNumElements(); I != E; ++I) { |
| 108 | Constant *Elt = CV->getElementAsConstant(I); |
| 109 | if (!match(Elt, m_APInt(IVal)) || !IVal->isPowerOf2()) |
| 110 | return 0; |
| 111 | Elts.push_back(ConstantInt::get(Elt->getType(), IVal->logBase2())); |
| 112 | } |
| 113 | |
| 114 | return ConstantVector::get(Elts); |
| 115 | } |
| 116 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 117 | Instruction *InstCombiner::visitMul(BinaryOperator &I) { |
Duncan Sands | 096aa79 | 2010-11-13 15:10:37 +0000 | [diff] [blame] | 118 | bool Changed = SimplifyAssociativeOrCommutative(I); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 119 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 120 | |
Duncan Sands | 82fdab3 | 2010-12-21 14:00:22 +0000 | [diff] [blame] | 121 | if (Value *V = SimplifyMulInst(Op0, Op1, TD)) |
| 122 | return ReplaceInstUsesWith(I, V); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 123 | |
Duncan Sands | 37bf92b | 2010-12-22 13:36:08 +0000 | [diff] [blame] | 124 | if (Value *V = SimplifyUsingDistributiveLaws(I)) |
| 125 | return ReplaceInstUsesWith(I, V); |
| 126 | |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 127 | if (match(Op1, m_AllOnes())) // X * -1 == 0 - X |
| 128 | return BinaryOperator::CreateNeg(Op0, I.getName()); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 129 | |
Rafael Espindola | 4f3d7ee | 2013-05-31 14:27:15 +0000 | [diff] [blame] | 130 | // Also allow combining multiply instructions on vectors. |
| 131 | { |
| 132 | Value *NewOp; |
| 133 | Constant *C1, *C2; |
| 134 | const APInt *IVal; |
| 135 | if (match(&I, m_Mul(m_Shl(m_Value(NewOp), m_Constant(C2)), |
| 136 | m_Constant(C1))) && |
| 137 | match(C1, m_APInt(IVal))) |
| 138 | // ((X << C1)*C2) == (X * (C2 << C1)) |
| 139 | return BinaryOperator::CreateMul(NewOp, ConstantExpr::getShl(C1, C2)); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 140 | |
Rafael Espindola | 4f3d7ee | 2013-05-31 14:27:15 +0000 | [diff] [blame] | 141 | if (match(&I, m_Mul(m_Value(NewOp), m_Constant(C1)))) { |
| 142 | Constant *NewCst = 0; |
| 143 | if (match(C1, m_APInt(IVal)) && IVal->isPowerOf2()) |
| 144 | // Replace X*(2^C) with X << C, where C is either a scalar or a splat. |
| 145 | NewCst = ConstantInt::get(NewOp->getType(), IVal->logBase2()); |
| 146 | else if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(C1)) |
| 147 | // Replace X*(2^C) with X << C, where C is a vector of known |
| 148 | // constant powers of 2. |
| 149 | NewCst = getLogBase2Vector(CV); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 150 | |
Rafael Espindola | 4f3d7ee | 2013-05-31 14:27:15 +0000 | [diff] [blame] | 151 | if (NewCst) { |
| 152 | BinaryOperator *Shl = BinaryOperator::CreateShl(NewOp, NewCst); |
| 153 | if (I.hasNoSignedWrap()) Shl->setHasNoSignedWrap(); |
| 154 | if (I.hasNoUnsignedWrap()) Shl->setHasNoUnsignedWrap(); |
| 155 | return Shl; |
| 156 | } |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 157 | } |
Rafael Espindola | 4f3d7ee | 2013-05-31 14:27:15 +0000 | [diff] [blame] | 158 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 159 | |
Rafael Espindola | 4f3d7ee | 2013-05-31 14:27:15 +0000 | [diff] [blame] | 160 | if (ConstantInt *CI = dyn_cast<ConstantInt>(Op1)) { |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 161 | // Canonicalize (X+C1)*CI -> X*CI+C1*CI. |
| 162 | { Value *X; ConstantInt *C1; |
| 163 | if (Op0->hasOneUse() && |
| 164 | match(Op0, m_Add(m_Value(X), m_ConstantInt(C1)))) { |
Benjamin Kramer | a9390a4 | 2011-09-27 20:39:19 +0000 | [diff] [blame] | 165 | Value *Add = Builder->CreateMul(X, CI); |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 166 | return BinaryOperator::CreateAdd(Add, Builder->CreateMul(C1, CI)); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 167 | } |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 168 | } |
Stuart Hastings | acbf107 | 2011-05-30 20:00:33 +0000 | [diff] [blame] | 169 | |
Stuart Hastings | f100282 | 2011-06-01 16:42:47 +0000 | [diff] [blame] | 170 | // (Y - X) * (-(2**n)) -> (X - Y) * (2**n), for positive nonzero n |
| 171 | // (Y + const) * (-(2**n)) -> (-constY) * (2**n), for positive nonzero n |
| 172 | // The "* (2**n)" thus becomes a potential shifting opportunity. |
Stuart Hastings | acbf107 | 2011-05-30 20:00:33 +0000 | [diff] [blame] | 173 | { |
| 174 | const APInt & Val = CI->getValue(); |
| 175 | const APInt &PosVal = Val.abs(); |
| 176 | if (Val.isNegative() && PosVal.isPowerOf2()) { |
Stuart Hastings | f100282 | 2011-06-01 16:42:47 +0000 | [diff] [blame] | 177 | Value *X = 0, *Y = 0; |
| 178 | if (Op0->hasOneUse()) { |
| 179 | ConstantInt *C1; |
| 180 | Value *Sub = 0; |
| 181 | if (match(Op0, m_Sub(m_Value(Y), m_Value(X)))) |
| 182 | Sub = Builder->CreateSub(X, Y, "suba"); |
| 183 | else if (match(Op0, m_Add(m_Value(Y), m_ConstantInt(C1)))) |
| 184 | Sub = Builder->CreateSub(Builder->CreateNeg(C1), Y, "subc"); |
| 185 | if (Sub) |
| 186 | return |
| 187 | BinaryOperator::CreateMul(Sub, |
| 188 | ConstantInt::get(Y->getType(), PosVal)); |
Stuart Hastings | acbf107 | 2011-05-30 20:00:33 +0000 | [diff] [blame] | 189 | } |
| 190 | } |
| 191 | } |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 192 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 193 | |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 194 | // Simplify mul instructions with a constant RHS. |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 195 | if (isa<Constant>(Op1)) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 196 | // Try to fold constant mul into select arguments. |
| 197 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0)) |
| 198 | if (Instruction *R = FoldOpIntoSelect(I, SI)) |
| 199 | return R; |
| 200 | |
| 201 | if (isa<PHINode>(Op0)) |
| 202 | if (Instruction *NV = FoldOpIntoPhi(I)) |
| 203 | return NV; |
| 204 | } |
| 205 | |
| 206 | if (Value *Op0v = dyn_castNegVal(Op0)) // -X * -Y = X*Y |
| 207 | if (Value *Op1v = dyn_castNegVal(Op1)) |
| 208 | return BinaryOperator::CreateMul(Op0v, Op1v); |
| 209 | |
| 210 | // (X / Y) * Y = X - (X % Y) |
| 211 | // (X / Y) * -Y = (X % Y) - X |
| 212 | { |
| 213 | Value *Op1C = Op1; |
| 214 | BinaryOperator *BO = dyn_cast<BinaryOperator>(Op0); |
| 215 | if (!BO || |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 216 | (BO->getOpcode() != Instruction::UDiv && |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 217 | BO->getOpcode() != Instruction::SDiv)) { |
| 218 | Op1C = Op0; |
| 219 | BO = dyn_cast<BinaryOperator>(Op1); |
| 220 | } |
| 221 | Value *Neg = dyn_castNegVal(Op1C); |
| 222 | if (BO && BO->hasOneUse() && |
| 223 | (BO->getOperand(1) == Op1C || BO->getOperand(1) == Neg) && |
| 224 | (BO->getOpcode() == Instruction::UDiv || |
| 225 | BO->getOpcode() == Instruction::SDiv)) { |
| 226 | Value *Op0BO = BO->getOperand(0), *Op1BO = BO->getOperand(1); |
| 227 | |
Chris Lattner | 35bda89 | 2011-02-06 21:44:57 +0000 | [diff] [blame] | 228 | // If the division is exact, X % Y is zero, so we end up with X or -X. |
| 229 | if (PossiblyExactOperator *SDiv = dyn_cast<PossiblyExactOperator>(BO)) |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 230 | if (SDiv->isExact()) { |
| 231 | if (Op1BO == Op1C) |
| 232 | return ReplaceInstUsesWith(I, Op0BO); |
| 233 | return BinaryOperator::CreateNeg(Op0BO); |
| 234 | } |
| 235 | |
| 236 | Value *Rem; |
| 237 | if (BO->getOpcode() == Instruction::UDiv) |
| 238 | Rem = Builder->CreateURem(Op0BO, Op1BO); |
| 239 | else |
| 240 | Rem = Builder->CreateSRem(Op0BO, Op1BO); |
| 241 | Rem->takeName(BO); |
| 242 | |
| 243 | if (Op1BO == Op1C) |
| 244 | return BinaryOperator::CreateSub(Op0BO, Rem); |
| 245 | return BinaryOperator::CreateSub(Rem, Op0BO); |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | /// i1 mul -> i1 and. |
Duncan Sands | b0bc6c3 | 2010-02-15 16:12:20 +0000 | [diff] [blame] | 250 | if (I.getType()->isIntegerTy(1)) |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 251 | return BinaryOperator::CreateAnd(Op0, Op1); |
| 252 | |
| 253 | // X*(1 << Y) --> X << Y |
| 254 | // (1 << Y)*X --> X << Y |
| 255 | { |
| 256 | Value *Y; |
| 257 | if (match(Op0, m_Shl(m_One(), m_Value(Y)))) |
| 258 | return BinaryOperator::CreateShl(Op1, Y); |
| 259 | if (match(Op1, m_Shl(m_One(), m_Value(Y)))) |
| 260 | return BinaryOperator::CreateShl(Op0, Y); |
| 261 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 262 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 263 | // If one of the operands of the multiply is a cast from a boolean value, then |
| 264 | // we know the bool is either zero or one, so this is a 'masking' multiply. |
| 265 | // X * Y (where Y is 0 or 1) -> X & (0-Y) |
Duncan Sands | 1df9859 | 2010-02-16 11:11:14 +0000 | [diff] [blame] | 266 | if (!I.getType()->isVectorTy()) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 267 | // -2 is "-1 << 1" so it is all bits set except the low one. |
| 268 | APInt Negative2(I.getType()->getPrimitiveSizeInBits(), (uint64_t)-2, true); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 269 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 270 | Value *BoolCast = 0, *OtherOp = 0; |
| 271 | if (MaskedValueIsZero(Op0, Negative2)) |
| 272 | BoolCast = Op0, OtherOp = Op1; |
| 273 | else if (MaskedValueIsZero(Op1, Negative2)) |
| 274 | BoolCast = Op1, OtherOp = Op0; |
| 275 | |
| 276 | if (BoolCast) { |
| 277 | Value *V = Builder->CreateSub(Constant::getNullValue(I.getType()), |
Benjamin Kramer | a9390a4 | 2011-09-27 20:39:19 +0000 | [diff] [blame] | 278 | BoolCast); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 279 | return BinaryOperator::CreateAnd(V, OtherOp); |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | return Changed ? &I : 0; |
| 284 | } |
| 285 | |
Pedro Artigas | c2a08d2 | 2012-11-30 22:07:05 +0000 | [diff] [blame] | 286 | // |
| 287 | // Detect pattern: |
| 288 | // |
| 289 | // log2(Y*0.5) |
| 290 | // |
| 291 | // And check for corresponding fast math flags |
| 292 | // |
| 293 | |
| 294 | static void detectLog2OfHalf(Value *&Op, Value *&Y, IntrinsicInst *&Log2) { |
Pedro Artigas | ef2ef3e | 2012-11-30 22:47:15 +0000 | [diff] [blame] | 295 | |
| 296 | if (!Op->hasOneUse()) |
| 297 | return; |
| 298 | |
| 299 | IntrinsicInst *II = dyn_cast<IntrinsicInst>(Op); |
| 300 | if (!II) |
| 301 | return; |
| 302 | if (II->getIntrinsicID() != Intrinsic::log2 || !II->hasUnsafeAlgebra()) |
| 303 | return; |
| 304 | Log2 = II; |
| 305 | |
| 306 | Value *OpLog2Of = II->getArgOperand(0); |
| 307 | if (!OpLog2Of->hasOneUse()) |
| 308 | return; |
| 309 | |
| 310 | Instruction *I = dyn_cast<Instruction>(OpLog2Of); |
| 311 | if (!I) |
| 312 | return; |
| 313 | if (I->getOpcode() != Instruction::FMul || !I->hasUnsafeAlgebra()) |
| 314 | return; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 315 | |
Pedro Artigas | ef2ef3e | 2012-11-30 22:47:15 +0000 | [diff] [blame] | 316 | ConstantFP *CFP = dyn_cast<ConstantFP>(I->getOperand(0)); |
| 317 | if (CFP && CFP->isExactlyValue(0.5)) { |
| 318 | Y = I->getOperand(1); |
| 319 | return; |
| 320 | } |
| 321 | CFP = dyn_cast<ConstantFP>(I->getOperand(1)); |
| 322 | if (CFP && CFP->isExactlyValue(0.5)) |
| 323 | Y = I->getOperand(0); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 324 | } |
Pedro Artigas | c2a08d2 | 2012-11-30 22:07:05 +0000 | [diff] [blame] | 325 | |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 326 | /// Helper function of InstCombiner::visitFMul(BinaryOperator(). It returns |
| 327 | /// true iff the given value is FMul or FDiv with one and only one operand |
| 328 | /// being a normal constant (i.e. not Zero/NaN/Infinity). |
| 329 | static bool isFMulOrFDivWithConstant(Value *V) { |
| 330 | Instruction *I = dyn_cast<Instruction>(V); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 331 | if (!I || (I->getOpcode() != Instruction::FMul && |
Shuxin Yang | f279731 | 2013-01-07 22:41:28 +0000 | [diff] [blame] | 332 | I->getOpcode() != Instruction::FDiv)) |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 333 | return false; |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 334 | |
| 335 | ConstantFP *C0 = dyn_cast<ConstantFP>(I->getOperand(0)); |
| 336 | ConstantFP *C1 = dyn_cast<ConstantFP>(I->getOperand(1)); |
| 337 | |
| 338 | if (C0 && C1) |
| 339 | return false; |
| 340 | |
Michael Gottesman | 07969dc | 2013-06-19 21:23:18 +0000 | [diff] [blame] | 341 | return (C0 && C0->getValueAPF().isFiniteNonZero()) || |
| 342 | (C1 && C1->getValueAPF().isFiniteNonZero()); |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 343 | } |
| 344 | |
| 345 | static bool isNormalFp(const ConstantFP *C) { |
| 346 | const APFloat &Flt = C->getValueAPF(); |
Michael Gottesman | c3cfe53 | 2013-06-26 23:17:31 +0000 | [diff] [blame] | 347 | return Flt.isNormal(); |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 348 | } |
| 349 | |
| 350 | /// foldFMulConst() is a helper routine of InstCombiner::visitFMul(). |
| 351 | /// The input \p FMulOrDiv is a FMul/FDiv with one and only one operand |
| 352 | /// being a constant (i.e. isFMulOrFDivWithConstant(FMulOrDiv) == true). |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 353 | /// This function is to simplify "FMulOrDiv * C" and returns the |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 354 | /// resulting expression. Note that this function could return NULL in |
| 355 | /// case the constants cannot be folded into a normal floating-point. |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 356 | /// |
Shuxin Yang | f279731 | 2013-01-07 22:41:28 +0000 | [diff] [blame] | 357 | Value *InstCombiner::foldFMulConst(Instruction *FMulOrDiv, ConstantFP *C, |
| 358 | Instruction *InsertBefore) { |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 359 | assert(isFMulOrFDivWithConstant(FMulOrDiv) && "V is invalid"); |
| 360 | |
| 361 | Value *Opnd0 = FMulOrDiv->getOperand(0); |
| 362 | Value *Opnd1 = FMulOrDiv->getOperand(1); |
| 363 | |
| 364 | ConstantFP *C0 = dyn_cast<ConstantFP>(Opnd0); |
| 365 | ConstantFP *C1 = dyn_cast<ConstantFP>(Opnd1); |
| 366 | |
| 367 | BinaryOperator *R = 0; |
| 368 | |
| 369 | // (X * C0) * C => X * (C0*C) |
| 370 | if (FMulOrDiv->getOpcode() == Instruction::FMul) { |
| 371 | Constant *F = ConstantExpr::getFMul(C1 ? C1 : C0, C); |
| 372 | if (isNormalFp(cast<ConstantFP>(F))) |
| 373 | R = BinaryOperator::CreateFMul(C1 ? Opnd0 : Opnd1, F); |
| 374 | } else { |
| 375 | if (C0) { |
| 376 | // (C0 / X) * C => (C0 * C) / X |
| 377 | ConstantFP *F = cast<ConstantFP>(ConstantExpr::getFMul(C0, C)); |
| 378 | if (isNormalFp(F)) |
| 379 | R = BinaryOperator::CreateFDiv(F, Opnd1); |
| 380 | } else { |
| 381 | // (X / C1) * C => X * (C/C1) if C/C1 is not a denormal |
| 382 | ConstantFP *F = cast<ConstantFP>(ConstantExpr::getFDiv(C, C1)); |
| 383 | if (isNormalFp(F)) { |
| 384 | R = BinaryOperator::CreateFMul(Opnd0, F); |
| 385 | } else { |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 386 | // (X / C1) * C => X / (C1/C) |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 387 | Constant *F = ConstantExpr::getFDiv(C1, C); |
| 388 | if (isNormalFp(cast<ConstantFP>(F))) |
| 389 | R = BinaryOperator::CreateFDiv(Opnd0, F); |
| 390 | } |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | if (R) { |
| 395 | R->setHasUnsafeAlgebra(true); |
| 396 | InsertNewInstWith(R, *InsertBefore); |
| 397 | } |
| 398 | |
| 399 | return R; |
| 400 | } |
| 401 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 402 | Instruction *InstCombiner::visitFMul(BinaryOperator &I) { |
Duncan Sands | 096aa79 | 2010-11-13 15:10:37 +0000 | [diff] [blame] | 403 | bool Changed = SimplifyAssociativeOrCommutative(I); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 404 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 405 | |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 406 | if (isa<Constant>(Op0)) |
| 407 | std::swap(Op0, Op1); |
| 408 | |
Michael Ilseman | c244f38 | 2012-12-12 00:28:32 +0000 | [diff] [blame] | 409 | if (Value *V = SimplifyFMulInst(Op0, Op1, I.getFastMathFlags(), TD)) |
| 410 | return ReplaceInstUsesWith(I, V); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 411 | |
Shuxin Yang | a144421 | 2013-01-15 21:09:32 +0000 | [diff] [blame] | 412 | bool AllowReassociate = I.hasUnsafeAlgebra(); |
| 413 | |
Michael Ilseman | c244f38 | 2012-12-12 00:28:32 +0000 | [diff] [blame] | 414 | // Simplify mul instructions with a constant RHS. |
| 415 | if (isa<Constant>(Op1)) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 416 | // Try to fold constant mul into select arguments. |
| 417 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0)) |
| 418 | if (Instruction *R = FoldOpIntoSelect(I, SI)) |
| 419 | return R; |
| 420 | |
| 421 | if (isa<PHINode>(Op0)) |
| 422 | if (Instruction *NV = FoldOpIntoPhi(I)) |
| 423 | return NV; |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 424 | |
| 425 | ConstantFP *C = dyn_cast<ConstantFP>(Op1); |
Michael Gottesman | 07969dc | 2013-06-19 21:23:18 +0000 | [diff] [blame] | 426 | if (C && AllowReassociate && C->getValueAPF().isFiniteNonZero()) { |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 427 | // Let MDC denote an expression in one of these forms: |
| 428 | // X * C, C/X, X/C, where C is a constant. |
| 429 | // |
| 430 | // Try to simplify "MDC * Constant" |
| 431 | if (isFMulOrFDivWithConstant(Op0)) { |
| 432 | Value *V = foldFMulConst(cast<Instruction>(Op0), C, &I); |
| 433 | if (V) |
| 434 | return ReplaceInstUsesWith(I, V); |
| 435 | } |
| 436 | |
Quentin Colombet | c5a4c25 | 2013-02-28 21:12:40 +0000 | [diff] [blame] | 437 | // (MDC +/- C1) * C => (MDC * C) +/- (C1 * C) |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 438 | Instruction *FAddSub = dyn_cast<Instruction>(Op0); |
| 439 | if (FAddSub && |
| 440 | (FAddSub->getOpcode() == Instruction::FAdd || |
| 441 | FAddSub->getOpcode() == Instruction::FSub)) { |
| 442 | Value *Opnd0 = FAddSub->getOperand(0); |
| 443 | Value *Opnd1 = FAddSub->getOperand(1); |
| 444 | ConstantFP *C0 = dyn_cast<ConstantFP>(Opnd0); |
| 445 | ConstantFP *C1 = dyn_cast<ConstantFP>(Opnd1); |
| 446 | bool Swap = false; |
| 447 | if (C0) { |
Shuxin Yang | f279731 | 2013-01-07 22:41:28 +0000 | [diff] [blame] | 448 | std::swap(C0, C1); |
| 449 | std::swap(Opnd0, Opnd1); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 450 | Swap = true; |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 451 | } |
| 452 | |
Michael Gottesman | 07969dc | 2013-06-19 21:23:18 +0000 | [diff] [blame] | 453 | if (C1 && C1->getValueAPF().isFiniteNonZero() && |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 454 | isFMulOrFDivWithConstant(Opnd0)) { |
Quentin Colombet | c5a4c25 | 2013-02-28 21:12:40 +0000 | [diff] [blame] | 455 | Value *M1 = ConstantExpr::getFMul(C1, C); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 456 | Value *M0 = isNormalFp(cast<ConstantFP>(M1)) ? |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 457 | foldFMulConst(cast<Instruction>(Opnd0), C, &I) : |
| 458 | 0; |
| 459 | if (M0 && M1) { |
| 460 | if (Swap && FAddSub->getOpcode() == Instruction::FSub) |
| 461 | std::swap(M0, M1); |
| 462 | |
| 463 | Value *R = (FAddSub->getOpcode() == Instruction::FAdd) ? |
| 464 | BinaryOperator::CreateFAdd(M0, M1) : |
| 465 | BinaryOperator::CreateFSub(M0, M1); |
| 466 | Instruction *RI = cast<Instruction>(R); |
Shuxin Yang | a144421 | 2013-01-15 21:09:32 +0000 | [diff] [blame] | 467 | RI->copyFastMathFlags(&I); |
Shuxin Yang | d3ae286 | 2013-01-07 21:39:23 +0000 | [diff] [blame] | 468 | return RI; |
| 469 | } |
| 470 | } |
| 471 | } |
| 472 | } |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 473 | } |
| 474 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 475 | |
Pedro Artigas | 84030dc | 2012-11-30 19:09:41 +0000 | [diff] [blame] | 476 | // Under unsafe algebra do: |
| 477 | // X * log2(0.5*Y) = X*log2(Y) - X |
| 478 | if (I.hasUnsafeAlgebra()) { |
| 479 | Value *OpX = NULL; |
| 480 | Value *OpY = NULL; |
| 481 | IntrinsicInst *Log2; |
Pedro Artigas | c2a08d2 | 2012-11-30 22:07:05 +0000 | [diff] [blame] | 482 | detectLog2OfHalf(Op0, OpY, Log2); |
| 483 | if (OpY) { |
| 484 | OpX = Op1; |
| 485 | } else { |
| 486 | detectLog2OfHalf(Op1, OpY, Log2); |
| 487 | if (OpY) { |
| 488 | OpX = Op0; |
Pedro Artigas | 84030dc | 2012-11-30 19:09:41 +0000 | [diff] [blame] | 489 | } |
| 490 | } |
| 491 | // if pattern detected emit alternate sequence |
| 492 | if (OpX && OpY) { |
| 493 | Log2->setArgOperand(0, OpY); |
| 494 | Value *FMulVal = Builder->CreateFMul(OpX, Log2); |
Pedro Artigas | c2a08d2 | 2012-11-30 22:07:05 +0000 | [diff] [blame] | 495 | Instruction *FMul = cast<Instruction>(FMulVal); |
Pedro Artigas | 84030dc | 2012-11-30 19:09:41 +0000 | [diff] [blame] | 496 | FMul->copyFastMathFlags(Log2); |
| 497 | Instruction *FSub = BinaryOperator::CreateFSub(FMulVal, OpX); |
| 498 | FSub->copyFastMathFlags(Log2); |
| 499 | return FSub; |
| 500 | } |
| 501 | } |
| 502 | |
Shuxin Yang | a144421 | 2013-01-15 21:09:32 +0000 | [diff] [blame] | 503 | // Handle symmetric situation in a 2-iteration loop |
| 504 | Value *Opnd0 = Op0; |
| 505 | Value *Opnd1 = Op1; |
| 506 | for (int i = 0; i < 2; i++) { |
| 507 | bool IgnoreZeroSign = I.hasNoSignedZeros(); |
| 508 | if (BinaryOperator::isFNeg(Opnd0, IgnoreZeroSign)) { |
| 509 | Value *N0 = dyn_castFNegVal(Opnd0, IgnoreZeroSign); |
| 510 | Value *N1 = dyn_castFNegVal(Opnd1, IgnoreZeroSign); |
Shuxin Yang | a5ed031 | 2012-12-14 18:46:06 +0000 | [diff] [blame] | 511 | |
Shuxin Yang | a144421 | 2013-01-15 21:09:32 +0000 | [diff] [blame] | 512 | // -X * -Y => X*Y |
| 513 | if (N1) |
| 514 | return BinaryOperator::CreateFMul(N0, N1); |
Shuxin Yang | a5ed031 | 2012-12-14 18:46:06 +0000 | [diff] [blame] | 515 | |
Shuxin Yang | a144421 | 2013-01-15 21:09:32 +0000 | [diff] [blame] | 516 | if (Opnd0->hasOneUse()) { |
| 517 | // -X * Y => -(X*Y) (Promote negation as high as possible) |
| 518 | Value *T = Builder->CreateFMul(N0, Opnd1); |
| 519 | cast<Instruction>(T)->setDebugLoc(I.getDebugLoc()); |
| 520 | Instruction *Neg = BinaryOperator::CreateFNeg(T); |
| 521 | if (I.getFastMathFlags().any()) { |
| 522 | cast<Instruction>(T)->copyFastMathFlags(&I); |
| 523 | Neg->copyFastMathFlags(&I); |
| 524 | } |
| 525 | return Neg; |
Shuxin Yang | a5ed031 | 2012-12-14 18:46:06 +0000 | [diff] [blame] | 526 | } |
| 527 | } |
Shuxin Yang | a144421 | 2013-01-15 21:09:32 +0000 | [diff] [blame] | 528 | |
| 529 | // (X*Y) * X => (X*X) * Y where Y != X |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 530 | // The purpose is two-fold: |
Shuxin Yang | a144421 | 2013-01-15 21:09:32 +0000 | [diff] [blame] | 531 | // 1) to form a power expression (of X). |
| 532 | // 2) potentially shorten the critical path: After transformation, the |
| 533 | // latency of the instruction Y is amortized by the expression of X*X, |
| 534 | // and therefore Y is in a "less critical" position compared to what it |
| 535 | // was before the transformation. |
| 536 | // |
| 537 | if (AllowReassociate) { |
| 538 | Value *Opnd0_0, *Opnd0_1; |
| 539 | if (Opnd0->hasOneUse() && |
| 540 | match(Opnd0, m_FMul(m_Value(Opnd0_0), m_Value(Opnd0_1)))) { |
| 541 | Value *Y = 0; |
| 542 | if (Opnd0_0 == Opnd1 && Opnd0_1 != Opnd1) |
| 543 | Y = Opnd0_1; |
| 544 | else if (Opnd0_1 == Opnd1 && Opnd0_0 != Opnd1) |
| 545 | Y = Opnd0_0; |
| 546 | |
| 547 | if (Y) { |
| 548 | Instruction *T = cast<Instruction>(Builder->CreateFMul(Opnd1, Opnd1)); |
| 549 | T->copyFastMathFlags(&I); |
| 550 | T->setDebugLoc(I.getDebugLoc()); |
| 551 | |
| 552 | Instruction *R = BinaryOperator::CreateFMul(T, Y); |
| 553 | R->copyFastMathFlags(&I); |
| 554 | return R; |
| 555 | } |
| 556 | } |
| 557 | } |
| 558 | |
| 559 | if (!isa<Constant>(Op1)) |
| 560 | std::swap(Opnd0, Opnd1); |
| 561 | else |
| 562 | break; |
Shuxin Yang | a5ed031 | 2012-12-14 18:46:06 +0000 | [diff] [blame] | 563 | } |
| 564 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 565 | return Changed ? &I : 0; |
| 566 | } |
| 567 | |
| 568 | /// SimplifyDivRemOfSelect - Try to fold a divide or remainder of a select |
| 569 | /// instruction. |
| 570 | bool InstCombiner::SimplifyDivRemOfSelect(BinaryOperator &I) { |
| 571 | SelectInst *SI = cast<SelectInst>(I.getOperand(1)); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 572 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 573 | // div/rem X, (Cond ? 0 : Y) -> div/rem X, Y |
| 574 | int NonNullOperand = -1; |
| 575 | if (Constant *ST = dyn_cast<Constant>(SI->getOperand(1))) |
| 576 | if (ST->isNullValue()) |
| 577 | NonNullOperand = 2; |
| 578 | // div/rem X, (Cond ? Y : 0) -> div/rem X, Y |
| 579 | if (Constant *ST = dyn_cast<Constant>(SI->getOperand(2))) |
| 580 | if (ST->isNullValue()) |
| 581 | NonNullOperand = 1; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 582 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 583 | if (NonNullOperand == -1) |
| 584 | return false; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 585 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 586 | Value *SelectCond = SI->getOperand(0); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 587 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 588 | // Change the div/rem to use 'Y' instead of the select. |
| 589 | I.setOperand(1, SI->getOperand(NonNullOperand)); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 590 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 591 | // Okay, we know we replace the operand of the div/rem with 'Y' with no |
| 592 | // problem. However, the select, or the condition of the select may have |
| 593 | // multiple uses. Based on our knowledge that the operand must be non-zero, |
| 594 | // propagate the known value for the select into other uses of it, and |
| 595 | // propagate a known value of the condition into its other users. |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 596 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 597 | // If the select and condition only have a single use, don't bother with this, |
| 598 | // early exit. |
| 599 | if (SI->use_empty() && SelectCond->hasOneUse()) |
| 600 | return true; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 601 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 602 | // Scan the current block backward, looking for other uses of SI. |
| 603 | BasicBlock::iterator BBI = &I, BBFront = I.getParent()->begin(); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 604 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 605 | while (BBI != BBFront) { |
| 606 | --BBI; |
| 607 | // If we found a call to a function, we can't assume it will return, so |
| 608 | // information from below it cannot be propagated above it. |
| 609 | if (isa<CallInst>(BBI) && !isa<IntrinsicInst>(BBI)) |
| 610 | break; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 611 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 612 | // Replace uses of the select or its condition with the known values. |
| 613 | for (Instruction::op_iterator I = BBI->op_begin(), E = BBI->op_end(); |
| 614 | I != E; ++I) { |
| 615 | if (*I == SI) { |
| 616 | *I = SI->getOperand(NonNullOperand); |
| 617 | Worklist.Add(BBI); |
| 618 | } else if (*I == SelectCond) { |
Jakub Staszak | 6a72c84 | 2013-06-06 23:34:59 +0000 | [diff] [blame] | 619 | *I = Builder->getInt1(NonNullOperand == 1); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 620 | Worklist.Add(BBI); |
| 621 | } |
| 622 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 623 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 624 | // If we past the instruction, quit looking for it. |
| 625 | if (&*BBI == SI) |
| 626 | SI = 0; |
| 627 | if (&*BBI == SelectCond) |
| 628 | SelectCond = 0; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 629 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 630 | // If we ran out of things to eliminate, break out of the loop. |
| 631 | if (SelectCond == 0 && SI == 0) |
| 632 | break; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 633 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 634 | } |
| 635 | return true; |
| 636 | } |
| 637 | |
| 638 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 639 | /// This function implements the transforms common to both integer division |
| 640 | /// instructions (udiv and sdiv). It is called by the visitors to those integer |
| 641 | /// division instructions. |
| 642 | /// @brief Common integer divide transforms |
| 643 | Instruction *InstCombiner::commonIDivTransforms(BinaryOperator &I) { |
| 644 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 645 | |
Chris Lattner | 1add46d | 2011-05-22 18:18:41 +0000 | [diff] [blame] | 646 | // The RHS is known non-zero. |
| 647 | if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this)) { |
| 648 | I.setOperand(1, V); |
| 649 | return &I; |
| 650 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 651 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 652 | // Handle cases involving: [su]div X, (select Cond, Y, Z) |
| 653 | // This does not apply for fdiv. |
| 654 | if (isa<SelectInst>(Op1) && SimplifyDivRemOfSelect(I)) |
| 655 | return &I; |
| 656 | |
| 657 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Op1)) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 658 | // (X / C1) / C2 -> X / (C1*C2) |
| 659 | if (Instruction *LHS = dyn_cast<Instruction>(Op0)) |
| 660 | if (Instruction::BinaryOps(LHS->getOpcode()) == I.getOpcode()) |
| 661 | if (ConstantInt *LHSRHS = dyn_cast<ConstantInt>(LHS->getOperand(1))) { |
| 662 | if (MultiplyOverflows(RHS, LHSRHS, |
| 663 | I.getOpcode()==Instruction::SDiv)) |
| 664 | return ReplaceInstUsesWith(I, Constant::getNullValue(I.getType())); |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 665 | return BinaryOperator::Create(I.getOpcode(), LHS->getOperand(0), |
| 666 | ConstantExpr::getMul(RHS, LHSRHS)); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 667 | } |
| 668 | |
| 669 | if (!RHS->isZero()) { // avoid X udiv 0 |
| 670 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0)) |
| 671 | if (Instruction *R = FoldOpIntoSelect(I, SI)) |
| 672 | return R; |
| 673 | if (isa<PHINode>(Op0)) |
| 674 | if (Instruction *NV = FoldOpIntoPhi(I)) |
| 675 | return NV; |
| 676 | } |
| 677 | } |
| 678 | |
Benjamin Kramer | 23b02cd | 2011-04-30 18:16:00 +0000 | [diff] [blame] | 679 | // See if we can fold away this div instruction. |
| 680 | if (SimplifyDemandedInstructionBits(I)) |
| 681 | return &I; |
| 682 | |
Duncan Sands | 593faa5 | 2011-01-28 16:51:11 +0000 | [diff] [blame] | 683 | // (X - (X rem Y)) / Y -> X / Y; usually originates as ((X / Y) * Y) / Y |
| 684 | Value *X = 0, *Z = 0; |
| 685 | if (match(Op0, m_Sub(m_Value(X), m_Value(Z)))) { // (X - Z) / Y; Y = Op1 |
| 686 | bool isSigned = I.getOpcode() == Instruction::SDiv; |
| 687 | if ((isSigned && match(Z, m_SRem(m_Specific(X), m_Specific(Op1)))) || |
| 688 | (!isSigned && match(Z, m_URem(m_Specific(X), m_Specific(Op1))))) |
| 689 | return BinaryOperator::Create(I.getOpcode(), X, Op1); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 690 | } |
| 691 | |
| 692 | return 0; |
| 693 | } |
| 694 | |
Benjamin Kramer | 7d6eb5a | 2011-04-30 18:16:07 +0000 | [diff] [blame] | 695 | /// dyn_castZExtVal - Checks if V is a zext or constant that can |
| 696 | /// be truncated to Ty without losing bits. |
Chris Lattner | db125cf | 2011-07-18 04:54:35 +0000 | [diff] [blame] | 697 | static Value *dyn_castZExtVal(Value *V, Type *Ty) { |
Benjamin Kramer | 7d6eb5a | 2011-04-30 18:16:07 +0000 | [diff] [blame] | 698 | if (ZExtInst *Z = dyn_cast<ZExtInst>(V)) { |
| 699 | if (Z->getSrcTy() == Ty) |
| 700 | return Z->getOperand(0); |
| 701 | } else if (ConstantInt *C = dyn_cast<ConstantInt>(V)) { |
| 702 | if (C->getValue().getActiveBits() <= cast<IntegerType>(Ty)->getBitWidth()) |
| 703 | return ConstantExpr::getTrunc(C, Ty); |
| 704 | } |
| 705 | return 0; |
| 706 | } |
| 707 | |
David Majnemer | e7006bb | 2013-07-04 21:17:49 +0000 | [diff] [blame] | 708 | namespace { |
| 709 | const unsigned MaxDepth = 6; |
| 710 | typedef Instruction *(*FoldUDivOperandCb)(Value *Op0, Value *Op1, |
| 711 | const BinaryOperator &I, |
| 712 | InstCombiner &IC); |
| 713 | |
| 714 | /// \brief Used to maintain state for visitUDivOperand(). |
| 715 | struct UDivFoldAction { |
| 716 | FoldUDivOperandCb FoldAction; ///< Informs visitUDiv() how to fold this |
| 717 | ///< operand. This can be zero if this action |
| 718 | ///< joins two actions together. |
| 719 | |
| 720 | Value *OperandToFold; ///< Which operand to fold. |
| 721 | union { |
| 722 | Instruction *FoldResult; ///< The instruction returned when FoldAction is |
| 723 | ///< invoked. |
| 724 | |
| 725 | size_t SelectLHSIdx; ///< Stores the LHS action index if this action |
| 726 | ///< joins two actions together. |
| 727 | }; |
| 728 | |
| 729 | UDivFoldAction(FoldUDivOperandCb FA, Value *InputOperand) |
| 730 | : FoldAction(FA), OperandToFold(InputOperand), FoldResult(0) {} |
| 731 | UDivFoldAction(FoldUDivOperandCb FA, Value *InputOperand, size_t SLHS) |
| 732 | : FoldAction(FA), OperandToFold(InputOperand), SelectLHSIdx(SLHS) {} |
| 733 | }; |
| 734 | } |
| 735 | |
| 736 | // X udiv 2^C -> X >> C |
| 737 | static Instruction *foldUDivPow2Cst(Value *Op0, Value *Op1, |
| 738 | const BinaryOperator &I, InstCombiner &IC) { |
| 739 | const APInt &C = cast<Constant>(Op1)->getUniqueInteger(); |
| 740 | BinaryOperator *LShr = BinaryOperator::CreateLShr( |
| 741 | Op0, ConstantInt::get(Op0->getType(), C.logBase2())); |
| 742 | if (I.isExact()) LShr->setIsExact(); |
| 743 | return LShr; |
| 744 | } |
| 745 | |
| 746 | // X udiv C, where C >= signbit |
| 747 | static Instruction *foldUDivNegCst(Value *Op0, Value *Op1, |
| 748 | const BinaryOperator &I, InstCombiner &IC) { |
| 749 | Value *ICI = IC.Builder->CreateICmpULT(Op0, cast<ConstantInt>(Op1)); |
| 750 | |
| 751 | return SelectInst::Create(ICI, Constant::getNullValue(I.getType()), |
| 752 | ConstantInt::get(I.getType(), 1)); |
| 753 | } |
| 754 | |
| 755 | // X udiv (C1 << N), where C1 is "1<<C2" --> X >> (N+C2) |
| 756 | static Instruction *foldUDivShl(Value *Op0, Value *Op1, const BinaryOperator &I, |
| 757 | InstCombiner &IC) { |
| 758 | Instruction *ShiftLeft = cast<Instruction>(Op1); |
| 759 | if (isa<ZExtInst>(ShiftLeft)) |
| 760 | ShiftLeft = cast<Instruction>(ShiftLeft->getOperand(0)); |
| 761 | |
| 762 | const APInt &CI = |
| 763 | cast<Constant>(ShiftLeft->getOperand(0))->getUniqueInteger(); |
| 764 | Value *N = ShiftLeft->getOperand(1); |
| 765 | if (CI != 1) |
| 766 | N = IC.Builder->CreateAdd(N, ConstantInt::get(N->getType(), CI.logBase2())); |
| 767 | if (ZExtInst *Z = dyn_cast<ZExtInst>(Op1)) |
| 768 | N = IC.Builder->CreateZExt(N, Z->getDestTy()); |
| 769 | BinaryOperator *LShr = BinaryOperator::CreateLShr(Op0, N); |
| 770 | if (I.isExact()) LShr->setIsExact(); |
| 771 | return LShr; |
| 772 | } |
| 773 | |
| 774 | // \brief Recursively visits the possible right hand operands of a udiv |
| 775 | // instruction, seeing through select instructions, to determine if we can |
| 776 | // replace the udiv with something simpler. If we find that an operand is not |
| 777 | // able to simplify the udiv, we abort the entire transformation. |
| 778 | static size_t visitUDivOperand(Value *Op0, Value *Op1, const BinaryOperator &I, |
| 779 | SmallVectorImpl<UDivFoldAction> &Actions, |
| 780 | unsigned Depth = 0) { |
| 781 | // Check to see if this is an unsigned division with an exact power of 2, |
| 782 | // if so, convert to a right shift. |
| 783 | if (match(Op1, m_Power2())) { |
| 784 | Actions.push_back(UDivFoldAction(foldUDivPow2Cst, Op1)); |
| 785 | return Actions.size(); |
| 786 | } |
| 787 | |
| 788 | if (ConstantInt *C = dyn_cast<ConstantInt>(Op1)) |
| 789 | // X udiv C, where C >= signbit |
| 790 | if (C->getValue().isNegative()) { |
| 791 | Actions.push_back(UDivFoldAction(foldUDivNegCst, C)); |
| 792 | return Actions.size(); |
| 793 | } |
| 794 | |
| 795 | // X udiv (C1 << N), where C1 is "1<<C2" --> X >> (N+C2) |
| 796 | if (match(Op1, m_Shl(m_Power2(), m_Value())) || |
| 797 | match(Op1, m_ZExt(m_Shl(m_Power2(), m_Value())))) { |
| 798 | Actions.push_back(UDivFoldAction(foldUDivShl, Op1)); |
| 799 | return Actions.size(); |
| 800 | } |
| 801 | |
| 802 | // The remaining tests are all recursive, so bail out if we hit the limit. |
| 803 | if (Depth++ == MaxDepth) |
| 804 | return 0; |
| 805 | |
| 806 | if (SelectInst *SI = dyn_cast<SelectInst>(Op1)) |
| 807 | if (size_t LHSIdx = visitUDivOperand(Op0, SI->getOperand(1), I, Actions)) |
| 808 | if (visitUDivOperand(Op0, SI->getOperand(2), I, Actions)) { |
| 809 | Actions.push_back(UDivFoldAction((FoldUDivOperandCb)0, Op1, LHSIdx-1)); |
| 810 | return Actions.size(); |
| 811 | } |
| 812 | |
| 813 | return 0; |
| 814 | } |
| 815 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 816 | Instruction *InstCombiner::visitUDiv(BinaryOperator &I) { |
| 817 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 818 | |
Duncan Sands | 593faa5 | 2011-01-28 16:51:11 +0000 | [diff] [blame] | 819 | if (Value *V = SimplifyUDivInst(Op0, Op1, TD)) |
| 820 | return ReplaceInstUsesWith(I, V); |
| 821 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 822 | // Handle the integer div common cases |
| 823 | if (Instruction *Common = commonIDivTransforms(I)) |
| 824 | return Common; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 825 | |
Benjamin Kramer | c81fe9c | 2012-08-30 15:07:40 +0000 | [diff] [blame] | 826 | // (x lshr C1) udiv C2 --> x udiv (C2 << C1) |
Nadav Rotem | a694e2a | 2012-08-28 12:23:22 +0000 | [diff] [blame] | 827 | if (ConstantInt *C2 = dyn_cast<ConstantInt>(Op1)) { |
Benjamin Kramer | aac7c65 | 2012-08-28 13:08:13 +0000 | [diff] [blame] | 828 | Value *X; |
| 829 | ConstantInt *C1; |
| 830 | if (match(Op0, m_LShr(m_Value(X), m_ConstantInt(C1)))) { |
Benjamin Kramer | 37dca63 | 2012-08-28 13:59:23 +0000 | [diff] [blame] | 831 | APInt NC = C2->getValue().shl(C1->getLimitedValue(C1->getBitWidth()-1)); |
Benjamin Kramer | aac7c65 | 2012-08-28 13:08:13 +0000 | [diff] [blame] | 832 | return BinaryOperator::CreateUDiv(X, Builder->getInt(NC)); |
Nadav Rotem | 9753f0b | 2012-08-28 10:01:43 +0000 | [diff] [blame] | 833 | } |
| 834 | } |
| 835 | |
Benjamin Kramer | 7d6eb5a | 2011-04-30 18:16:07 +0000 | [diff] [blame] | 836 | // (zext A) udiv (zext B) --> zext (A udiv B) |
| 837 | if (ZExtInst *ZOp0 = dyn_cast<ZExtInst>(Op0)) |
| 838 | if (Value *ZOp1 = dyn_castZExtVal(Op1, ZOp0->getSrcTy())) |
| 839 | return new ZExtInst(Builder->CreateUDiv(ZOp0->getOperand(0), ZOp1, "div", |
| 840 | I.isExact()), |
| 841 | I.getType()); |
| 842 | |
David Majnemer | e7006bb | 2013-07-04 21:17:49 +0000 | [diff] [blame] | 843 | // (LHS udiv (select (select (...)))) -> (LHS >> (select (select (...)))) |
| 844 | SmallVector<UDivFoldAction, 6> UDivActions; |
| 845 | if (visitUDivOperand(Op0, Op1, I, UDivActions)) |
| 846 | for (unsigned i = 0, e = UDivActions.size(); i != e; ++i) { |
| 847 | FoldUDivOperandCb Action = UDivActions[i].FoldAction; |
| 848 | Value *ActionOp1 = UDivActions[i].OperandToFold; |
| 849 | Instruction *Inst; |
| 850 | if (Action) |
| 851 | Inst = Action(Op0, ActionOp1, I, *this); |
| 852 | else { |
| 853 | // This action joins two actions together. The RHS of this action is |
| 854 | // simply the last action we processed, we saved the LHS action index in |
| 855 | // the joining action. |
| 856 | size_t SelectRHSIdx = i - 1; |
| 857 | Value *SelectRHS = UDivActions[SelectRHSIdx].FoldResult; |
| 858 | size_t SelectLHSIdx = UDivActions[i].SelectLHSIdx; |
| 859 | Value *SelectLHS = UDivActions[SelectLHSIdx].FoldResult; |
| 860 | Inst = SelectInst::Create(cast<SelectInst>(ActionOp1)->getCondition(), |
| 861 | SelectLHS, SelectRHS); |
| 862 | } |
| 863 | |
| 864 | // If this is the last action to process, return it to the InstCombiner. |
| 865 | // Otherwise, we insert it before the UDiv and record it so that we may |
| 866 | // use it as part of a joining action (i.e., a SelectInst). |
| 867 | if (e - i != 1) { |
| 868 | Inst->insertBefore(&I); |
| 869 | UDivActions[i].FoldResult = Inst; |
| 870 | } else |
| 871 | return Inst; |
| 872 | } |
| 873 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 874 | return 0; |
| 875 | } |
| 876 | |
| 877 | Instruction *InstCombiner::visitSDiv(BinaryOperator &I) { |
| 878 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 879 | |
Duncan Sands | 593faa5 | 2011-01-28 16:51:11 +0000 | [diff] [blame] | 880 | if (Value *V = SimplifySDivInst(Op0, Op1, TD)) |
| 881 | return ReplaceInstUsesWith(I, V); |
| 882 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 883 | // Handle the integer div common cases |
| 884 | if (Instruction *Common = commonIDivTransforms(I)) |
| 885 | return Common; |
| 886 | |
| 887 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Op1)) { |
| 888 | // sdiv X, -1 == -X |
| 889 | if (RHS->isAllOnesValue()) |
| 890 | return BinaryOperator::CreateNeg(Op0); |
| 891 | |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 892 | // sdiv X, C --> ashr exact X, log2(C) |
| 893 | if (I.isExact() && RHS->getValue().isNonNegative() && |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 894 | RHS->getValue().isPowerOf2()) { |
| 895 | Value *ShAmt = llvm::ConstantInt::get(RHS->getType(), |
| 896 | RHS->getValue().exactLogBase2()); |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 897 | return BinaryOperator::CreateExactAShr(Op0, ShAmt, I.getName()); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 898 | } |
| 899 | |
| 900 | // -X/C --> X/-C provided the negation doesn't overflow. |
| 901 | if (SubOperator *Sub = dyn_cast<SubOperator>(Op0)) |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 902 | if (match(Sub->getOperand(0), m_Zero()) && Sub->hasNoSignedWrap()) |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 903 | return BinaryOperator::CreateSDiv(Sub->getOperand(1), |
| 904 | ConstantExpr::getNeg(RHS)); |
| 905 | } |
| 906 | |
| 907 | // If the sign bits of both operands are zero (i.e. we can prove they are |
| 908 | // unsigned inputs), turn this into a udiv. |
Duncan Sands | b0bc6c3 | 2010-02-15 16:12:20 +0000 | [diff] [blame] | 909 | if (I.getType()->isIntegerTy()) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 910 | APInt Mask(APInt::getSignBit(I.getType()->getPrimitiveSizeInBits())); |
| 911 | if (MaskedValueIsZero(Op0, Mask)) { |
| 912 | if (MaskedValueIsZero(Op1, Mask)) { |
Sylvestre Ledru | 94c2271 | 2012-09-27 10:14:43 +0000 | [diff] [blame] | 913 | // X sdiv Y -> X udiv Y, iff X and Y don't have sign bit set |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 914 | return BinaryOperator::CreateUDiv(Op0, Op1, I.getName()); |
| 915 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 916 | |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 917 | if (match(Op1, m_Shl(m_Power2(), m_Value()))) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 918 | // X sdiv (1 << Y) -> X udiv (1 << Y) ( -> X u>> Y) |
| 919 | // Safe because the only negative value (1 << Y) can take on is |
| 920 | // INT_MIN, and X sdiv INT_MIN == X udiv INT_MIN == 0 if X doesn't have |
| 921 | // the sign bit set. |
| 922 | return BinaryOperator::CreateUDiv(Op0, Op1, I.getName()); |
| 923 | } |
| 924 | } |
| 925 | } |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 926 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 927 | return 0; |
| 928 | } |
| 929 | |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 930 | /// CvtFDivConstToReciprocal tries to convert X/C into X*1/C if C not a special |
| 931 | /// FP value and: |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 932 | /// 1) 1/C is exact, or |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 933 | /// 2) reciprocal is allowed. |
Sylvestre Ledru | da2ed45 | 2013-05-14 23:36:24 +0000 | [diff] [blame] | 934 | /// If the conversion was successful, the simplified expression "X * 1/C" is |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 935 | /// returned; otherwise, NULL is returned. |
| 936 | /// |
| 937 | static Instruction *CvtFDivConstToReciprocal(Value *Dividend, |
| 938 | ConstantFP *Divisor, |
| 939 | bool AllowReciprocal) { |
| 940 | const APFloat &FpVal = Divisor->getValueAPF(); |
| 941 | APFloat Reciprocal(FpVal.getSemantics()); |
| 942 | bool Cvt = FpVal.getExactInverse(&Reciprocal); |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 943 | |
Michael Gottesman | 07969dc | 2013-06-19 21:23:18 +0000 | [diff] [blame] | 944 | if (!Cvt && AllowReciprocal && FpVal.isFiniteNonZero()) { |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 945 | Reciprocal = APFloat(FpVal.getSemantics(), 1.0f); |
| 946 | (void)Reciprocal.divide(FpVal, APFloat::rmNearestTiesToEven); |
| 947 | Cvt = !Reciprocal.isDenormal(); |
| 948 | } |
| 949 | |
| 950 | if (!Cvt) |
| 951 | return 0; |
| 952 | |
| 953 | ConstantFP *R; |
| 954 | R = ConstantFP::get(Dividend->getType()->getContext(), Reciprocal); |
| 955 | return BinaryOperator::CreateFMul(Dividend, R); |
| 956 | } |
| 957 | |
Frits van Bommel | 31726c1 | 2011-01-29 17:50:27 +0000 | [diff] [blame] | 958 | Instruction *InstCombiner::visitFDiv(BinaryOperator &I) { |
| 959 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 960 | |
| 961 | if (Value *V = SimplifyFDivInst(Op0, Op1, TD)) |
| 962 | return ReplaceInstUsesWith(I, V); |
| 963 | |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 964 | bool AllowReassociate = I.hasUnsafeAlgebra(); |
| 965 | bool AllowReciprocal = I.hasAllowReciprocal(); |
Benjamin Kramer | 5467396 | 2011-03-30 15:42:35 +0000 | [diff] [blame] | 966 | |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 967 | if (ConstantFP *Op1C = dyn_cast<ConstantFP>(Op1)) { |
| 968 | if (AllowReassociate) { |
| 969 | ConstantFP *C1 = 0; |
| 970 | ConstantFP *C2 = Op1C; |
| 971 | Value *X; |
| 972 | Instruction *Res = 0; |
| 973 | |
| 974 | if (match(Op0, m_FMul(m_Value(X), m_ConstantFP(C1)))) { |
| 975 | // (X*C1)/C2 => X * (C1/C2) |
| 976 | // |
| 977 | Constant *C = ConstantExpr::getFDiv(C1, C2); |
| 978 | const APFloat &F = cast<ConstantFP>(C)->getValueAPF(); |
Michael Gottesman | c3cfe53 | 2013-06-26 23:17:31 +0000 | [diff] [blame] | 979 | if (F.isNormal()) |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 980 | Res = BinaryOperator::CreateFMul(X, C); |
| 981 | } else if (match(Op0, m_FDiv(m_Value(X), m_ConstantFP(C1)))) { |
| 982 | // (X/C1)/C2 => X /(C2*C1) [=> X * 1/(C2*C1) if reciprocal is allowed] |
| 983 | // |
| 984 | Constant *C = ConstantExpr::getFMul(C1, C2); |
| 985 | const APFloat &F = cast<ConstantFP>(C)->getValueAPF(); |
Michael Gottesman | c3cfe53 | 2013-06-26 23:17:31 +0000 | [diff] [blame] | 986 | if (F.isNormal()) { |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 987 | Res = CvtFDivConstToReciprocal(X, cast<ConstantFP>(C), |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 988 | AllowReciprocal); |
| 989 | if (!Res) |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 990 | Res = BinaryOperator::CreateFDiv(X, C); |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 991 | } |
| 992 | } |
| 993 | |
| 994 | if (Res) { |
| 995 | Res->setFastMathFlags(I.getFastMathFlags()); |
| 996 | return Res; |
| 997 | } |
| 998 | } |
| 999 | |
| 1000 | // X / C => X * 1/C |
| 1001 | if (Instruction *T = CvtFDivConstToReciprocal(Op0, Op1C, AllowReciprocal)) |
| 1002 | return T; |
| 1003 | |
| 1004 | return 0; |
| 1005 | } |
| 1006 | |
| 1007 | if (AllowReassociate && isa<ConstantFP>(Op0)) { |
| 1008 | ConstantFP *C1 = cast<ConstantFP>(Op0), *C2; |
| 1009 | Constant *Fold = 0; |
| 1010 | Value *X; |
| 1011 | bool CreateDiv = true; |
| 1012 | |
| 1013 | // C1 / (X*C2) => (C1/C2) / X |
| 1014 | if (match(Op1, m_FMul(m_Value(X), m_ConstantFP(C2)))) |
| 1015 | Fold = ConstantExpr::getFDiv(C1, C2); |
| 1016 | else if (match(Op1, m_FDiv(m_Value(X), m_ConstantFP(C2)))) { |
| 1017 | // C1 / (X/C2) => (C1*C2) / X |
| 1018 | Fold = ConstantExpr::getFMul(C1, C2); |
| 1019 | } else if (match(Op1, m_FDiv(m_ConstantFP(C2), m_Value(X)))) { |
| 1020 | // C1 / (C2/X) => (C1/C2) * X |
| 1021 | Fold = ConstantExpr::getFDiv(C1, C2); |
| 1022 | CreateDiv = false; |
| 1023 | } |
| 1024 | |
| 1025 | if (Fold) { |
| 1026 | const APFloat &FoldC = cast<ConstantFP>(Fold)->getValueAPF(); |
Michael Gottesman | c3cfe53 | 2013-06-26 23:17:31 +0000 | [diff] [blame] | 1027 | if (FoldC.isNormal()) { |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 1028 | Instruction *R = CreateDiv ? |
Shuxin Yang | 7d72cf8 | 2013-01-14 22:48:41 +0000 | [diff] [blame] | 1029 | BinaryOperator::CreateFDiv(Fold, X) : |
| 1030 | BinaryOperator::CreateFMul(X, Fold); |
| 1031 | R->setFastMathFlags(I.getFastMathFlags()); |
| 1032 | return R; |
| 1033 | } |
| 1034 | } |
| 1035 | return 0; |
| 1036 | } |
| 1037 | |
| 1038 | if (AllowReassociate) { |
| 1039 | Value *X, *Y; |
| 1040 | Value *NewInst = 0; |
| 1041 | Instruction *SimpR = 0; |
| 1042 | |
| 1043 | if (Op0->hasOneUse() && match(Op0, m_FDiv(m_Value(X), m_Value(Y)))) { |
| 1044 | // (X/Y) / Z => X / (Y*Z) |
| 1045 | // |
| 1046 | if (!isa<ConstantFP>(Y) || !isa<ConstantFP>(Op1)) { |
| 1047 | NewInst = Builder->CreateFMul(Y, Op1); |
| 1048 | SimpR = BinaryOperator::CreateFDiv(X, NewInst); |
| 1049 | } |
| 1050 | } else if (Op1->hasOneUse() && match(Op1, m_FDiv(m_Value(X), m_Value(Y)))) { |
| 1051 | // Z / (X/Y) => Z*Y / X |
| 1052 | // |
| 1053 | if (!isa<ConstantFP>(Y) || !isa<ConstantFP>(Op0)) { |
| 1054 | NewInst = Builder->CreateFMul(Op0, Y); |
| 1055 | SimpR = BinaryOperator::CreateFDiv(NewInst, X); |
| 1056 | } |
| 1057 | } |
| 1058 | |
| 1059 | if (NewInst) { |
| 1060 | if (Instruction *T = dyn_cast<Instruction>(NewInst)) |
| 1061 | T->setDebugLoc(I.getDebugLoc()); |
| 1062 | SimpR->setFastMathFlags(I.getFastMathFlags()); |
| 1063 | return SimpR; |
Benjamin Kramer | 5467396 | 2011-03-30 15:42:35 +0000 | [diff] [blame] | 1064 | } |
| 1065 | } |
| 1066 | |
Frits van Bommel | 31726c1 | 2011-01-29 17:50:27 +0000 | [diff] [blame] | 1067 | return 0; |
| 1068 | } |
| 1069 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1070 | /// This function implements the transforms common to both integer remainder |
| 1071 | /// instructions (urem and srem). It is called by the visitors to those integer |
| 1072 | /// remainder instructions. |
| 1073 | /// @brief Common integer remainder transforms |
| 1074 | Instruction *InstCombiner::commonIRemTransforms(BinaryOperator &I) { |
| 1075 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 1076 | |
Chris Lattner | 1add46d | 2011-05-22 18:18:41 +0000 | [diff] [blame] | 1077 | // The RHS is known non-zero. |
| 1078 | if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this)) { |
| 1079 | I.setOperand(1, V); |
| 1080 | return &I; |
| 1081 | } |
| 1082 | |
Duncan Sands | f24ed77 | 2011-05-02 16:27:02 +0000 | [diff] [blame] | 1083 | // Handle cases involving: rem X, (select Cond, Y, Z) |
| 1084 | if (isa<SelectInst>(Op1) && SimplifyDivRemOfSelect(I)) |
| 1085 | return &I; |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1086 | |
Duncan Sands | 00676a6 | 2011-05-02 18:41:29 +0000 | [diff] [blame] | 1087 | if (isa<ConstantInt>(Op1)) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1088 | if (Instruction *Op0I = dyn_cast<Instruction>(Op0)) { |
| 1089 | if (SelectInst *SI = dyn_cast<SelectInst>(Op0I)) { |
| 1090 | if (Instruction *R = FoldOpIntoSelect(I, SI)) |
| 1091 | return R; |
| 1092 | } else if (isa<PHINode>(Op0I)) { |
| 1093 | if (Instruction *NV = FoldOpIntoPhi(I)) |
| 1094 | return NV; |
| 1095 | } |
| 1096 | |
| 1097 | // See if we can fold away this rem instruction. |
| 1098 | if (SimplifyDemandedInstructionBits(I)) |
| 1099 | return &I; |
| 1100 | } |
| 1101 | } |
| 1102 | |
| 1103 | return 0; |
| 1104 | } |
| 1105 | |
| 1106 | Instruction *InstCombiner::visitURem(BinaryOperator &I) { |
| 1107 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 1108 | |
Duncan Sands | f24ed77 | 2011-05-02 16:27:02 +0000 | [diff] [blame] | 1109 | if (Value *V = SimplifyURemInst(Op0, Op1, TD)) |
| 1110 | return ReplaceInstUsesWith(I, V); |
| 1111 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1112 | if (Instruction *common = commonIRemTransforms(I)) |
| 1113 | return common; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 1114 | |
David Majnemer | fa49d7d | 2013-05-12 00:07:05 +0000 | [diff] [blame] | 1115 | // (zext A) urem (zext B) --> zext (A urem B) |
| 1116 | if (ZExtInst *ZOp0 = dyn_cast<ZExtInst>(Op0)) |
| 1117 | if (Value *ZOp1 = dyn_castZExtVal(Op1, ZOp0->getSrcTy())) |
| 1118 | return new ZExtInst(Builder->CreateURem(ZOp0->getOperand(0), ZOp1), |
| 1119 | I.getType()); |
| 1120 | |
David Majnemer | a8ccefc | 2013-05-11 09:01:28 +0000 | [diff] [blame] | 1121 | // X urem Y -> X and Y-1, where Y is a power of 2, |
| 1122 | if (isKnownToBeAPowerOfTwo(Op1, /*OrZero*/true)) { |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 1123 | Constant *N1 = Constant::getAllOnesValue(I.getType()); |
Benjamin Kramer | a9390a4 | 2011-09-27 20:39:19 +0000 | [diff] [blame] | 1124 | Value *Add = Builder->CreateAdd(Op1, N1); |
Chris Lattner | 7a6aa1a | 2011-02-10 05:36:31 +0000 | [diff] [blame] | 1125 | return BinaryOperator::CreateAnd(Op0, Add); |
| 1126 | } |
| 1127 | |
Nick Lewycky | 75681bb | 2013-07-13 01:16:47 +0000 | [diff] [blame^] | 1128 | // 1 urem X -> zext(X != 1) |
| 1129 | if (match(Op0, m_One())) { |
| 1130 | Value *Cmp = Builder->CreateICmpNE(Op1, Op0); |
| 1131 | Value *Ext = Builder->CreateZExt(Cmp, I.getType()); |
| 1132 | return ReplaceInstUsesWith(I, Ext); |
| 1133 | } |
| 1134 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1135 | return 0; |
| 1136 | } |
| 1137 | |
| 1138 | Instruction *InstCombiner::visitSRem(BinaryOperator &I) { |
| 1139 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
| 1140 | |
Duncan Sands | f24ed77 | 2011-05-02 16:27:02 +0000 | [diff] [blame] | 1141 | if (Value *V = SimplifySRemInst(Op0, Op1, TD)) |
| 1142 | return ReplaceInstUsesWith(I, V); |
| 1143 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1144 | // Handle the integer rem common cases |
| 1145 | if (Instruction *Common = commonIRemTransforms(I)) |
| 1146 | return Common; |
Jim Grosbach | 03fceff | 2013-04-05 21:20:12 +0000 | [diff] [blame] | 1147 | |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1148 | if (Value *RHSNeg = dyn_castNegVal(Op1)) |
| 1149 | if (!isa<Constant>(RHSNeg) || |
| 1150 | (isa<ConstantInt>(RHSNeg) && |
| 1151 | cast<ConstantInt>(RHSNeg)->getValue().isStrictlyPositive())) { |
| 1152 | // X % -Y -> X % Y |
| 1153 | Worklist.AddValue(I.getOperand(1)); |
| 1154 | I.setOperand(1, RHSNeg); |
| 1155 | return &I; |
| 1156 | } |
| 1157 | |
| 1158 | // If the sign bits of both operands are zero (i.e. we can prove they are |
| 1159 | // unsigned inputs), turn this into a urem. |
Duncan Sands | b0bc6c3 | 2010-02-15 16:12:20 +0000 | [diff] [blame] | 1160 | if (I.getType()->isIntegerTy()) { |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1161 | APInt Mask(APInt::getSignBit(I.getType()->getPrimitiveSizeInBits())); |
| 1162 | if (MaskedValueIsZero(Op1, Mask) && MaskedValueIsZero(Op0, Mask)) { |
Sylvestre Ledru | 94c2271 | 2012-09-27 10:14:43 +0000 | [diff] [blame] | 1163 | // X srem Y -> X urem Y, iff X and Y don't have sign bit set |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1164 | return BinaryOperator::CreateURem(Op0, Op1, I.getName()); |
| 1165 | } |
| 1166 | } |
| 1167 | |
| 1168 | // If it's a constant vector, flip any negative values positive. |
Chris Lattner | a78fa8c | 2012-01-27 03:08:05 +0000 | [diff] [blame] | 1169 | if (isa<ConstantVector>(Op1) || isa<ConstantDataVector>(Op1)) { |
| 1170 | Constant *C = cast<Constant>(Op1); |
| 1171 | unsigned VWidth = C->getType()->getVectorNumElements(); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1172 | |
| 1173 | bool hasNegative = false; |
Chris Lattner | a78fa8c | 2012-01-27 03:08:05 +0000 | [diff] [blame] | 1174 | bool hasMissing = false; |
| 1175 | for (unsigned i = 0; i != VWidth; ++i) { |
| 1176 | Constant *Elt = C->getAggregateElement(i); |
| 1177 | if (Elt == 0) { |
| 1178 | hasMissing = true; |
| 1179 | break; |
| 1180 | } |
| 1181 | |
| 1182 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Elt)) |
Chris Lattner | c73b24d | 2011-07-15 06:08:15 +0000 | [diff] [blame] | 1183 | if (RHS->isNegative()) |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1184 | hasNegative = true; |
Chris Lattner | a78fa8c | 2012-01-27 03:08:05 +0000 | [diff] [blame] | 1185 | } |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1186 | |
Chris Lattner | a78fa8c | 2012-01-27 03:08:05 +0000 | [diff] [blame] | 1187 | if (hasNegative && !hasMissing) { |
Chris Lattner | 4ca829e | 2012-01-25 06:02:56 +0000 | [diff] [blame] | 1188 | SmallVector<Constant *, 16> Elts(VWidth); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1189 | for (unsigned i = 0; i != VWidth; ++i) { |
Chris Lattner | 7302d80 | 2012-02-06 21:56:39 +0000 | [diff] [blame] | 1190 | Elts[i] = C->getAggregateElement(i); // Handle undef, etc. |
Chris Lattner | a78fa8c | 2012-01-27 03:08:05 +0000 | [diff] [blame] | 1191 | if (ConstantInt *RHS = dyn_cast<ConstantInt>(Elts[i])) { |
Chris Lattner | c73b24d | 2011-07-15 06:08:15 +0000 | [diff] [blame] | 1192 | if (RHS->isNegative()) |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1193 | Elts[i] = cast<ConstantInt>(ConstantExpr::getNeg(RHS)); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1194 | } |
| 1195 | } |
| 1196 | |
| 1197 | Constant *NewRHSV = ConstantVector::get(Elts); |
Chris Lattner | a78fa8c | 2012-01-27 03:08:05 +0000 | [diff] [blame] | 1198 | if (NewRHSV != C) { // Don't loop on -MININT |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1199 | Worklist.AddValue(I.getOperand(1)); |
| 1200 | I.setOperand(1, NewRHSV); |
| 1201 | return &I; |
| 1202 | } |
| 1203 | } |
| 1204 | } |
| 1205 | |
| 1206 | return 0; |
| 1207 | } |
| 1208 | |
| 1209 | Instruction *InstCombiner::visitFRem(BinaryOperator &I) { |
Duncan Sands | f24ed77 | 2011-05-02 16:27:02 +0000 | [diff] [blame] | 1210 | Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1); |
Chris Lattner | d12c27c | 2010-01-05 06:09:35 +0000 | [diff] [blame] | 1211 | |
Duncan Sands | f24ed77 | 2011-05-02 16:27:02 +0000 | [diff] [blame] | 1212 | if (Value *V = SimplifyFRemInst(Op0, Op1, TD)) |
| 1213 | return ReplaceInstUsesWith(I, V); |
| 1214 | |
| 1215 | // Handle cases involving: rem X, (select Cond, Y, Z) |
| 1216 | if (isa<SelectInst>(Op1) && SimplifyDivRemOfSelect(I)) |
| 1217 | return &I; |
| 1218 | |
| 1219 | return 0; |
| 1220 | } |