blob: 2013de065a0cdf1259deb98176cdf605eae5af21 [file] [log] [blame]
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001//===- InstCombineCalls.cpp -----------------------------------------------===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Chris Lattner7a9e47a2010-01-05 07:32:13 +00006//
7//===----------------------------------------------------------------------===//
8//
Craig Topper784929d2019-02-08 20:48:56 +00009// This file implements the visitCall, visitInvoke, and visitCallBr functions.
Chris Lattner7a9e47a2010-01-05 07:32:13 +000010//
11//===----------------------------------------------------------------------===//
12
Chandler Carrutha9174582015-01-22 05:25:13 +000013#include "InstCombineInternal.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000014#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/None.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000018#include "llvm/ADT/Optional.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000019#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
Chandler Carruth6bda14b2017-06-06 11:49:48 +000021#include "llvm/ADT/Statistic.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000022#include "llvm/ADT/Twine.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000023#include "llvm/Analysis/AssumptionCache.h"
David Majnemer15032582015-05-22 03:56:46 +000024#include "llvm/Analysis/InstructionSimplify.h"
Simon Pilgrim4b7d3c42019-04-25 09:49:37 +000025#include "llvm/Analysis/Loads.h"
Chris Lattner7a9e47a2010-01-05 07:32:13 +000026#include "llvm/Analysis/MemoryBuiltins.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000027#include "llvm/Analysis/ValueTracking.h"
Philip Reamese4588bb2019-03-20 18:44:58 +000028#include "llvm/Analysis/VectorUtils.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000029#include "llvm/IR/Attributes.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000030#include "llvm/IR/BasicBlock.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000031#include "llvm/IR/Constant.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000032#include "llvm/IR/Constants.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000033#include "llvm/IR/DataLayout.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/GlobalVariable.h"
37#include "llvm/IR/InstrTypes.h"
38#include "llvm/IR/Instruction.h"
39#include "llvm/IR/Instructions.h"
40#include "llvm/IR/IntrinsicInst.h"
41#include "llvm/IR/Intrinsics.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/Metadata.h"
Chandler Carruth820a9082014-03-04 11:08:18 +000044#include "llvm/IR/PatternMatch.h"
Philip Reames1a1bdb22014-12-02 18:50:36 +000045#include "llvm/IR/Statepoint.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000046#include "llvm/IR/Type.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000047#include "llvm/IR/User.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000048#include "llvm/IR/Value.h"
49#include "llvm/IR/ValueHandle.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000050#include "llvm/Support/AtomicOrdering.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000051#include "llvm/Support/Casting.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000052#include "llvm/Support/CommandLine.h"
53#include "llvm/Support/Compiler.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000054#include "llvm/Support/Debug.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000055#include "llvm/Support/ErrorHandling.h"
Craig Topperb45eabc2017-04-26 16:39:58 +000056#include "llvm/Support/KnownBits.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000057#include "llvm/Support/MathExtras.h"
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000058#include "llvm/Support/raw_ostream.h"
59#include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
Philip Reames88cd69b2019-04-25 02:30:17 +000060#include "llvm/Transforms/Utils/Local.h"
Chandler Carruthba4c5172015-01-21 11:23:40 +000061#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
Eugene Zelenkocdc71612016-08-11 17:20:18 +000062#include <algorithm>
63#include <cassert>
64#include <cstdint>
65#include <cstring>
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +000066#include <utility>
Eugene Zelenkocdc71612016-08-11 17:20:18 +000067#include <vector>
68
Chris Lattner7a9e47a2010-01-05 07:32:13 +000069using namespace llvm;
Michael Ilseman536cc322012-12-13 03:13:36 +000070using namespace PatternMatch;
Chris Lattner7a9e47a2010-01-05 07:32:13 +000071
Chandler Carruth964daaa2014-04-22 02:55:47 +000072#define DEBUG_TYPE "instcombine"
73
Meador Ingee3f2b262012-11-30 04:05:06 +000074STATISTIC(NumSimplified, "Number of library calls simplified");
75
Philip Reames79e917d2018-05-09 22:56:32 +000076static cl::opt<unsigned> GuardWideningWindow(
77 "instcombine-guard-widening-window",
78 cl::init(3),
79 cl::desc("How wide an instruction window to bypass looking for "
80 "another guard"));
81
Sanjay Patelcd4377c2016-01-20 22:24:38 +000082/// Return the specified type promoted as it would be to pass though a va_arg
83/// area.
Chris Lattner229907c2011-07-18 04:54:35 +000084static Type *getPromotedType(Type *Ty) {
85 if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +000086 if (ITy->getBitWidth() < 32)
87 return Type::getInt32Ty(Ty->getContext());
88 }
89 return Ty;
90}
91
Sanjay Patel368ac5d2016-02-21 17:29:33 +000092/// Return a constant boolean vector that has true elements in all positions
Sanjay Patel24401302016-02-21 17:33:31 +000093/// where the input constant data vector has an element with the sign bit set.
Sanjay Patel368ac5d2016-02-21 17:29:33 +000094static Constant *getNegativeIsTrueBoolVec(ConstantDataVector *V) {
95 SmallVector<Constant *, 32> BoolVec;
96 IntegerType *BoolTy = Type::getInt1Ty(V->getContext());
97 for (unsigned I = 0, E = V->getNumElements(); I != E; ++I) {
98 Constant *Elt = V->getElementAsConstant(I);
99 assert((isa<ConstantInt>(Elt) || isa<ConstantFP>(Elt)) &&
100 "Unexpected constant data vector element type");
101 bool Sign = V->getElementType()->isIntegerTy()
102 ? cast<ConstantInt>(Elt)->isNegative()
103 : cast<ConstantFP>(Elt)->isNegative();
104 BoolVec.push_back(ConstantInt::get(BoolTy, Sign));
105 }
106 return ConstantVector::get(BoolVec);
107}
108
Daniel Neilson8f30ec62018-05-11 14:30:02 +0000109Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
Daniel Neilson2363da92018-02-12 23:06:55 +0000110 unsigned DstAlign = getKnownAlignment(MI->getRawDest(), DL, MI, &AC, &DT);
111 unsigned CopyDstAlign = MI->getDestAlignment();
112 if (CopyDstAlign < DstAlign){
113 MI->setDestAlignment(DstAlign);
114 return MI;
115 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000116
Daniel Neilson8f30ec62018-05-11 14:30:02 +0000117 unsigned SrcAlign = getKnownAlignment(MI->getRawSource(), DL, MI, &AC, &DT);
118 unsigned CopySrcAlign = MI->getSourceAlignment();
Daniel Neilson2363da92018-02-12 23:06:55 +0000119 if (CopySrcAlign < SrcAlign) {
Daniel Neilson8f30ec62018-05-11 14:30:02 +0000120 MI->setSourceAlignment(SrcAlign);
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000121 return MI;
122 }
Jim Grosbach7815f562012-02-03 00:07:04 +0000123
Philip Reamesd7486892019-04-22 20:28:19 +0000124 // If we have a store to a location which is known constant, we can conclude
125 // that the store must be storing the constant value (else the memory
126 // wouldn't be constant), and this must be a noop.
127 if (AA->pointsToConstantMemory(MI->getDest())) {
128 // Set the size of the copy to 0, it will be deleted on the next iteration.
129 MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
130 return MI;
131 }
132
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000133 // If MemCpyInst length is 1/2/4/8 bytes then replace memcpy with
134 // load/store.
Daniel Neilson8f30ec62018-05-11 14:30:02 +0000135 ConstantInt *MemOpLength = dyn_cast<ConstantInt>(MI->getLength());
Craig Topperf40110f2014-04-25 05:29:35 +0000136 if (!MemOpLength) return nullptr;
Jim Grosbach7815f562012-02-03 00:07:04 +0000137
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000138 // Source and destination pointer types are always "i8*" for intrinsic. See
139 // if the size is something we can handle with a single primitive load/store.
140 // A single load+store correctly handles overlapping memory in the memmove
141 // case.
Michael Liao69e172a2012-08-15 03:49:59 +0000142 uint64_t Size = MemOpLength->getLimitedValue();
Alp Tokercb402912014-01-24 17:20:08 +0000143 assert(Size && "0-sized memory transferring should be removed already.");
Jim Grosbach7815f562012-02-03 00:07:04 +0000144
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000145 if (Size > 8 || (Size&(Size-1)))
Craig Topperf40110f2014-04-25 05:29:35 +0000146 return nullptr; // If not 1/2/4/8 bytes, exit.
Jim Grosbach7815f562012-02-03 00:07:04 +0000147
Serguei Katkova5b0e552019-01-16 04:36:26 +0000148 // If it is an atomic and alignment is less than the size then we will
149 // introduce the unaligned memory access which will be later transformed
150 // into libcall in CodeGen. This is not evident performance gain so disable
151 // it now.
152 if (isa<AtomicMemTransferInst>(MI))
153 if (CopyDstAlign < Size || CopySrcAlign < Size)
154 return nullptr;
155
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000156 // Use an integer load+store unless we can find something better.
Mon P Wangc576ee92010-04-04 03:10:48 +0000157 unsigned SrcAddrSp =
Gabor Greif0a136c92010-06-24 13:54:33 +0000158 cast<PointerType>(MI->getArgOperand(1)->getType())->getAddressSpace();
Gabor Greiff3755202010-04-16 15:33:14 +0000159 unsigned DstAddrSp =
Gabor Greif0a136c92010-06-24 13:54:33 +0000160 cast<PointerType>(MI->getArgOperand(0)->getType())->getAddressSpace();
Mon P Wangc576ee92010-04-04 03:10:48 +0000161
Chris Lattner229907c2011-07-18 04:54:35 +0000162 IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
Mon P Wangc576ee92010-04-04 03:10:48 +0000163 Type *NewSrcPtrTy = PointerType::get(IntType, SrcAddrSp);
164 Type *NewDstPtrTy = PointerType::get(IntType, DstAddrSp);
Jim Grosbach7815f562012-02-03 00:07:04 +0000165
Mikael Holmen760dc9a2017-03-01 06:45:20 +0000166 // If the memcpy has metadata describing the members, see if we can get the
167 // TBAA tag describing our copy.
Craig Topperf40110f2014-04-25 05:29:35 +0000168 MDNode *CopyMD = nullptr;
Ivan A. Kosarevf03f5792018-02-19 12:10:20 +0000169 if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa)) {
170 CopyMD = M;
171 } else if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa_struct)) {
Mikael Holmen760dc9a2017-03-01 06:45:20 +0000172 if (M->getNumOperands() == 3 && M->getOperand(0) &&
173 mdconst::hasa<ConstantInt>(M->getOperand(0)) &&
Craig Topper79ab6432017-07-06 18:39:47 +0000174 mdconst::extract<ConstantInt>(M->getOperand(0))->isZero() &&
Mikael Holmen760dc9a2017-03-01 06:45:20 +0000175 M->getOperand(1) &&
176 mdconst::hasa<ConstantInt>(M->getOperand(1)) &&
177 mdconst::extract<ConstantInt>(M->getOperand(1))->getValue() ==
178 Size &&
179 M->getOperand(2) && isa<MDNode>(M->getOperand(2)))
180 CopyMD = cast<MDNode>(M->getOperand(2));
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000181 }
Jim Grosbach7815f562012-02-03 00:07:04 +0000182
Craig Topperbb4069e2017-07-07 23:16:26 +0000183 Value *Src = Builder.CreateBitCast(MI->getArgOperand(1), NewSrcPtrTy);
184 Value *Dest = Builder.CreateBitCast(MI->getArgOperand(0), NewDstPtrTy);
James Y Knight14359ef2019-02-01 20:44:24 +0000185 LoadInst *L = Builder.CreateLoad(IntType, Src);
Daniel Neilson2363da92018-02-12 23:06:55 +0000186 // Alignment from the mem intrinsic will be better, so use it.
187 L->setAlignment(CopySrcAlign);
Dan Gohman3f553c22012-09-13 21:51:01 +0000188 if (CopyMD)
189 L->setMetadata(LLVMContext::MD_tbaa, CopyMD);
Dorit Nuzmanabd15f62016-09-04 07:49:39 +0000190 MDNode *LoopMemParallelMD =
191 MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
192 if (LoopMemParallelMD)
193 L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
Michael Kruse978ba612018-12-20 04:58:07 +0000194 MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);
195 if (AccessGroupMD)
196 L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
Dorit Nuzman7673ba72016-09-04 07:06:00 +0000197
Daniel Neilson8f30ec62018-05-11 14:30:02 +0000198 StoreInst *S = Builder.CreateStore(L, Dest);
Daniel Neilson2363da92018-02-12 23:06:55 +0000199 // Alignment from the mem intrinsic will be better, so use it.
200 S->setAlignment(CopyDstAlign);
Dan Gohman3f553c22012-09-13 21:51:01 +0000201 if (CopyMD)
202 S->setMetadata(LLVMContext::MD_tbaa, CopyMD);
Dorit Nuzmanabd15f62016-09-04 07:49:39 +0000203 if (LoopMemParallelMD)
204 S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
Michael Kruse978ba612018-12-20 04:58:07 +0000205 if (AccessGroupMD)
206 S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000207
Daniel Neilson8f30ec62018-05-11 14:30:02 +0000208 if (auto *MT = dyn_cast<MemTransferInst>(MI)) {
209 // non-atomics can be volatile
210 L->setVolatile(MT->isVolatile());
211 S->setVolatile(MT->isVolatile());
212 }
213 if (isa<AtomicMemTransferInst>(MI)) {
214 // atomics have to be unordered
215 L->setOrdering(AtomicOrdering::Unordered);
216 S->setOrdering(AtomicOrdering::Unordered);
217 }
218
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000219 // Set the size of the copy to 0, it will be deleted on the next iteration.
Daniel Neilson8f30ec62018-05-11 14:30:02 +0000220 MI->setLength(Constant::getNullValue(MemOpLength->getType()));
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000221 return MI;
222}
223
Daniel Neilsonf6651d42018-05-11 20:04:50 +0000224Instruction *InstCombiner::SimplifyAnyMemSet(AnyMemSetInst *MI) {
Daniel Jasperaec2fa32016-12-19 08:22:17 +0000225 unsigned Alignment = getKnownAlignment(MI->getDest(), DL, MI, &AC, &DT);
Daniel Neilson38af2ee2018-02-02 22:03:03 +0000226 if (MI->getDestAlignment() < Alignment) {
227 MI->setDestAlignment(Alignment);
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000228 return MI;
229 }
Jim Grosbach7815f562012-02-03 00:07:04 +0000230
Philip Reamesd7486892019-04-22 20:28:19 +0000231 // If we have a store to a location which is known constant, we can conclude
232 // that the store must be storing the constant value (else the memory
233 // wouldn't be constant), and this must be a noop.
234 if (AA->pointsToConstantMemory(MI->getDest())) {
235 // Set the size of the copy to 0, it will be deleted on the next iteration.
236 MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
237 return MI;
238 }
239
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000240 // Extract the length and alignment and fill if they are constant.
241 ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());
242 ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue());
Duncan Sands9dff9be2010-02-15 16:12:20 +0000243 if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8))
Craig Topperf40110f2014-04-25 05:29:35 +0000244 return nullptr;
Michael Liao69e172a2012-08-15 03:49:59 +0000245 uint64_t Len = LenC->getLimitedValue();
Daniel Neilson710d7b92018-03-22 18:36:15 +0000246 Alignment = MI->getDestAlignment();
Michael Liao69e172a2012-08-15 03:49:59 +0000247 assert(Len && "0-sized memory setting should be removed already.");
Jim Grosbach7815f562012-02-03 00:07:04 +0000248
Serguei Katkova5b0e552019-01-16 04:36:26 +0000249 // Alignment 0 is identity for alignment 1 for memset, but not store.
250 if (Alignment == 0)
251 Alignment = 1;
252
253 // If it is an atomic and alignment is less than the size then we will
254 // introduce the unaligned memory access which will be later transformed
255 // into libcall in CodeGen. This is not evident performance gain so disable
256 // it now.
257 if (isa<AtomicMemSetInst>(MI))
258 if (Alignment < Len)
259 return nullptr;
260
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000261 // memset(s,c,n) -> store s, c (for n=1,2,4,8)
262 if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
Chris Lattner229907c2011-07-18 04:54:35 +0000263 Type *ITy = IntegerType::get(MI->getContext(), Len*8); // n=1 -> i8.
Jim Grosbach7815f562012-02-03 00:07:04 +0000264
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000265 Value *Dest = MI->getDest();
Mon P Wang1991c472010-12-20 01:05:30 +0000266 unsigned DstAddrSp = cast<PointerType>(Dest->getType())->getAddressSpace();
267 Type *NewDstPtrTy = PointerType::get(ITy, DstAddrSp);
Craig Topperbb4069e2017-07-07 23:16:26 +0000268 Dest = Builder.CreateBitCast(Dest, NewDstPtrTy);
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000269
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000270 // Extract the fill value and store.
271 uint64_t Fill = FillC->getZExtValue()*0x0101010101010101ULL;
Craig Topperbb4069e2017-07-07 23:16:26 +0000272 StoreInst *S = Builder.CreateStore(ConstantInt::get(ITy, Fill), Dest,
273 MI->isVolatile());
Eli Friedman49346012011-05-18 19:57:14 +0000274 S->setAlignment(Alignment);
Daniel Neilsonf6651d42018-05-11 20:04:50 +0000275 if (isa<AtomicMemSetInst>(MI))
276 S->setOrdering(AtomicOrdering::Unordered);
Jim Grosbach7815f562012-02-03 00:07:04 +0000277
Chris Lattner7a9e47a2010-01-05 07:32:13 +0000278 // Set the size of the copy to 0, it will be deleted on the next iteration.
279 MI->setLength(Constant::getNullValue(LenC->getType()));
280 return MI;
281 }
282
Simon Pilgrim18617d12015-08-05 08:18:00 +0000283 return nullptr;
284}
285
Sanjay Patel6038d3e2016-01-29 23:27:03 +0000286static Value *simplifyX86immShift(const IntrinsicInst &II,
Simon Pilgrimbecd5e82015-08-13 07:39:03 +0000287 InstCombiner::BuilderTy &Builder) {
288 bool LogicalShift = false;
289 bool ShiftLeft = false;
290
291 switch (II.getIntrinsicID()) {
Craig Topperb4173a52016-11-13 07:26:19 +0000292 default: llvm_unreachable("Unexpected intrinsic!");
Simon Pilgrimbecd5e82015-08-13 07:39:03 +0000293 case Intrinsic::x86_sse2_psra_d:
294 case Intrinsic::x86_sse2_psra_w:
295 case Intrinsic::x86_sse2_psrai_d:
296 case Intrinsic::x86_sse2_psrai_w:
297 case Intrinsic::x86_avx2_psra_d:
298 case Intrinsic::x86_avx2_psra_w:
299 case Intrinsic::x86_avx2_psrai_d:
300 case Intrinsic::x86_avx2_psrai_w:
Craig Topper8b831cb2016-11-13 01:51:55 +0000301 case Intrinsic::x86_avx512_psra_q_128:
302 case Intrinsic::x86_avx512_psrai_q_128:
303 case Intrinsic::x86_avx512_psra_q_256:
304 case Intrinsic::x86_avx512_psrai_q_256:
305 case Intrinsic::x86_avx512_psra_d_512:
306 case Intrinsic::x86_avx512_psra_q_512:
307 case Intrinsic::x86_avx512_psra_w_512:
308 case Intrinsic::x86_avx512_psrai_d_512:
309 case Intrinsic::x86_avx512_psrai_q_512:
310 case Intrinsic::x86_avx512_psrai_w_512:
Simon Pilgrimbecd5e82015-08-13 07:39:03 +0000311 LogicalShift = false; ShiftLeft = false;
312 break;
313 case Intrinsic::x86_sse2_psrl_d:
314 case Intrinsic::x86_sse2_psrl_q:
315 case Intrinsic::x86_sse2_psrl_w:
316 case Intrinsic::x86_sse2_psrli_d:
317 case Intrinsic::x86_sse2_psrli_q:
318 case Intrinsic::x86_sse2_psrli_w:
319 case Intrinsic::x86_avx2_psrl_d:
320 case Intrinsic::x86_avx2_psrl_q:
321 case Intrinsic::x86_avx2_psrl_w:
322 case Intrinsic::x86_avx2_psrli_d:
323 case Intrinsic::x86_avx2_psrli_q:
324 case Intrinsic::x86_avx2_psrli_w:
Craig Topper8b831cb2016-11-13 01:51:55 +0000325 case Intrinsic::x86_avx512_psrl_d_512:
326 case Intrinsic::x86_avx512_psrl_q_512:
327 case Intrinsic::x86_avx512_psrl_w_512:
328 case Intrinsic::x86_avx512_psrli_d_512:
329 case Intrinsic::x86_avx512_psrli_q_512:
330 case Intrinsic::x86_avx512_psrli_w_512:
Simon Pilgrimbecd5e82015-08-13 07:39:03 +0000331 LogicalShift = true; ShiftLeft = false;
332 break;
333 case Intrinsic::x86_sse2_psll_d:
334 case Intrinsic::x86_sse2_psll_q:
335 case Intrinsic::x86_sse2_psll_w:
336 case Intrinsic::x86_sse2_pslli_d:
337 case Intrinsic::x86_sse2_pslli_q:
338 case Intrinsic::x86_sse2_pslli_w:
339 case Intrinsic::x86_avx2_psll_d:
340 case Intrinsic::x86_avx2_psll_q:
341 case Intrinsic::x86_avx2_psll_w:
342 case Intrinsic::x86_avx2_pslli_d:
343 case Intrinsic::x86_avx2_pslli_q:
344 case Intrinsic::x86_avx2_pslli_w:
Craig Topper8b831cb2016-11-13 01:51:55 +0000345 case Intrinsic::x86_avx512_psll_d_512:
346 case Intrinsic::x86_avx512_psll_q_512:
347 case Intrinsic::x86_avx512_psll_w_512:
348 case Intrinsic::x86_avx512_pslli_d_512:
349 case Intrinsic::x86_avx512_pslli_q_512:
350 case Intrinsic::x86_avx512_pslli_w_512:
Simon Pilgrimbecd5e82015-08-13 07:39:03 +0000351 LogicalShift = true; ShiftLeft = true;
352 break;
353 }
Simon Pilgrima3a72b42015-08-10 20:21:15 +0000354 assert((LogicalShift || !ShiftLeft) && "Only logical shifts can shift left");
355
Simon Pilgrim3815c162015-08-07 18:22:50 +0000356 // Simplify if count is constant.
357 auto Arg1 = II.getArgOperand(1);
358 auto CAZ = dyn_cast<ConstantAggregateZero>(Arg1);
359 auto CDV = dyn_cast<ConstantDataVector>(Arg1);
360 auto CInt = dyn_cast<ConstantInt>(Arg1);
361 if (!CAZ && !CDV && !CInt)
Simon Pilgrim18617d12015-08-05 08:18:00 +0000362 return nullptr;
Simon Pilgrim3815c162015-08-07 18:22:50 +0000363
364 APInt Count(64, 0);
365 if (CDV) {
366 // SSE2/AVX2 uses all the first 64-bits of the 128-bit vector
367 // operand to compute the shift amount.
368 auto VT = cast<VectorType>(CDV->getType());
369 unsigned BitWidth = VT->getElementType()->getPrimitiveSizeInBits();
370 assert((64 % BitWidth) == 0 && "Unexpected packed shift size");
371 unsigned NumSubElts = 64 / BitWidth;
372
373 // Concatenate the sub-elements to create the 64-bit value.
374 for (unsigned i = 0; i != NumSubElts; ++i) {
375 unsigned SubEltIdx = (NumSubElts - 1) - i;
376 auto SubElt = cast<ConstantInt>(CDV->getElementAsConstant(SubEltIdx));
Craig Topper24e71012017-04-28 03:36:24 +0000377 Count <<= BitWidth;
Simon Pilgrim3815c162015-08-07 18:22:50 +0000378 Count |= SubElt->getValue().zextOrTrunc(64);
379 }
380 }
381 else if (CInt)
382 Count = CInt->getValue();
Simon Pilgrim18617d12015-08-05 08:18:00 +0000383
384 auto Vec = II.getArgOperand(0);
385 auto VT = cast<VectorType>(Vec->getType());
386 auto SVT = VT->getElementType();
Simon Pilgrim3815c162015-08-07 18:22:50 +0000387 unsigned VWidth = VT->getNumElements();
388 unsigned BitWidth = SVT->getPrimitiveSizeInBits();
389
390 // If shift-by-zero then just return the original value.
Craig Topper73ba1c82017-06-07 07:40:37 +0000391 if (Count.isNullValue())
Simon Pilgrim3815c162015-08-07 18:22:50 +0000392 return Vec;
393
Simon Pilgrima3a72b42015-08-10 20:21:15 +0000394 // Handle cases when Shift >= BitWidth.
395 if (Count.uge(BitWidth)) {
396 // If LogicalShift - just return zero.
397 if (LogicalShift)
398 return ConstantAggregateZero::get(VT);
399
400 // If ArithmeticShift - clamp Shift to (BitWidth - 1).
401 Count = APInt(64, BitWidth - 1);
402 }
Simon Pilgrim18617d12015-08-05 08:18:00 +0000403
Simon Pilgrim18617d12015-08-05 08:18:00 +0000404 // Get a constant vector of the same type as the first operand.
Simon Pilgrim3815c162015-08-07 18:22:50 +0000405 auto ShiftAmt = ConstantInt::get(SVT, Count.zextOrTrunc(BitWidth));
406 auto ShiftVec = Builder.CreateVectorSplat(VWidth, ShiftAmt);
Simon Pilgrim18617d12015-08-05 08:18:00 +0000407
408 if (ShiftLeft)
Simon Pilgrim3815c162015-08-07 18:22:50 +0000409 return Builder.CreateShl(Vec, ShiftVec);
Simon Pilgrim18617d12015-08-05 08:18:00 +0000410
Simon Pilgrima3a72b42015-08-10 20:21:15 +0000411 if (LogicalShift)
412 return Builder.CreateLShr(Vec, ShiftVec);
413
414 return Builder.CreateAShr(Vec, ShiftVec);
Simon Pilgrim18617d12015-08-05 08:18:00 +0000415}
416
Simon Pilgrimdb9893f2016-06-07 10:27:15 +0000417// Attempt to simplify AVX2 per-element shift intrinsics to a generic IR shift.
418// Unlike the generic IR shifts, the intrinsics have defined behaviour for out
419// of range shift amounts (logical - set to zero, arithmetic - splat sign bit).
420static Value *simplifyX86varShift(const IntrinsicInst &II,
421 InstCombiner::BuilderTy &Builder) {
422 bool LogicalShift = false;
423 bool ShiftLeft = false;
424
425 switch (II.getIntrinsicID()) {
Craig Topperb4173a52016-11-13 07:26:19 +0000426 default: llvm_unreachable("Unexpected intrinsic!");
Simon Pilgrimdb9893f2016-06-07 10:27:15 +0000427 case Intrinsic::x86_avx2_psrav_d:
428 case Intrinsic::x86_avx2_psrav_d_256:
Craig Topperb4173a52016-11-13 07:26:19 +0000429 case Intrinsic::x86_avx512_psrav_q_128:
430 case Intrinsic::x86_avx512_psrav_q_256:
431 case Intrinsic::x86_avx512_psrav_d_512:
432 case Intrinsic::x86_avx512_psrav_q_512:
Craig Topper1de753f2016-11-18 06:04:33 +0000433 case Intrinsic::x86_avx512_psrav_w_128:
434 case Intrinsic::x86_avx512_psrav_w_256:
435 case Intrinsic::x86_avx512_psrav_w_512:
Simon Pilgrimdb9893f2016-06-07 10:27:15 +0000436 LogicalShift = false;
437 ShiftLeft = false;
438 break;
439 case Intrinsic::x86_avx2_psrlv_d:
440 case Intrinsic::x86_avx2_psrlv_d_256:
441 case Intrinsic::x86_avx2_psrlv_q:
442 case Intrinsic::x86_avx2_psrlv_q_256:
Craig Topperb4173a52016-11-13 07:26:19 +0000443 case Intrinsic::x86_avx512_psrlv_d_512:
444 case Intrinsic::x86_avx512_psrlv_q_512:
Craig Topper1de753f2016-11-18 06:04:33 +0000445 case Intrinsic::x86_avx512_psrlv_w_128:
446 case Intrinsic::x86_avx512_psrlv_w_256:
447 case Intrinsic::x86_avx512_psrlv_w_512:
Simon Pilgrimdb9893f2016-06-07 10:27:15 +0000448 LogicalShift = true;
449 ShiftLeft = false;
450 break;
451 case Intrinsic::x86_avx2_psllv_d:
452 case Intrinsic::x86_avx2_psllv_d_256:
453 case Intrinsic::x86_avx2_psllv_q:
454 case Intrinsic::x86_avx2_psllv_q_256:
Craig Topperb4173a52016-11-13 07:26:19 +0000455 case Intrinsic::x86_avx512_psllv_d_512:
456 case Intrinsic::x86_avx512_psllv_q_512:
Craig Topper1de753f2016-11-18 06:04:33 +0000457 case Intrinsic::x86_avx512_psllv_w_128:
458 case Intrinsic::x86_avx512_psllv_w_256:
459 case Intrinsic::x86_avx512_psllv_w_512:
Simon Pilgrimdb9893f2016-06-07 10:27:15 +0000460 LogicalShift = true;
461 ShiftLeft = true;
462 break;
463 }
464 assert((LogicalShift || !ShiftLeft) && "Only logical shifts can shift left");
465
466 // Simplify if all shift amounts are constant/undef.
467 auto *CShift = dyn_cast<Constant>(II.getArgOperand(1));
468 if (!CShift)
469 return nullptr;
470
471 auto Vec = II.getArgOperand(0);
472 auto VT = cast<VectorType>(II.getType());
473 auto SVT = VT->getVectorElementType();
474 int NumElts = VT->getNumElements();
475 int BitWidth = SVT->getIntegerBitWidth();
476
477 // Collect each element's shift amount.
478 // We also collect special cases: UNDEF = -1, OUT-OF-RANGE = BitWidth.
479 bool AnyOutOfRange = false;
480 SmallVector<int, 8> ShiftAmts;
481 for (int I = 0; I < NumElts; ++I) {
482 auto *CElt = CShift->getAggregateElement(I);
483 if (CElt && isa<UndefValue>(CElt)) {
484 ShiftAmts.push_back(-1);
485 continue;
486 }
487
488 auto *COp = dyn_cast_or_null<ConstantInt>(CElt);
489 if (!COp)
490 return nullptr;
491
492 // Handle out of range shifts.
493 // If LogicalShift - set to BitWidth (special case).
494 // If ArithmeticShift - set to (BitWidth - 1) (sign splat).
495 APInt ShiftVal = COp->getValue();
496 if (ShiftVal.uge(BitWidth)) {
497 AnyOutOfRange = LogicalShift;
498 ShiftAmts.push_back(LogicalShift ? BitWidth : BitWidth - 1);
499 continue;
500 }
501
502 ShiftAmts.push_back((int)ShiftVal.getZExtValue());
503 }
504
505 // If all elements out of range or UNDEF, return vector of zeros/undefs.
506 // ArithmeticShift should only hit this if they are all UNDEF.
507 auto OutOfRange = [&](int Idx) { return (Idx < 0) || (BitWidth <= Idx); };
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +0000508 if (llvm::all_of(ShiftAmts, OutOfRange)) {
Simon Pilgrimdb9893f2016-06-07 10:27:15 +0000509 SmallVector<Constant *, 8> ConstantVec;
510 for (int Idx : ShiftAmts) {
511 if (Idx < 0) {
512 ConstantVec.push_back(UndefValue::get(SVT));
513 } else {
514 assert(LogicalShift && "Logical shift expected");
515 ConstantVec.push_back(ConstantInt::getNullValue(SVT));
516 }
517 }
518 return ConstantVector::get(ConstantVec);
519 }
520
521 // We can't handle only some out of range values with generic logical shifts.
522 if (AnyOutOfRange)
523 return nullptr;
524
525 // Build the shift amount constant vector.
526 SmallVector<Constant *, 8> ShiftVecAmts;
527 for (int Idx : ShiftAmts) {
528 if (Idx < 0)
529 ShiftVecAmts.push_back(UndefValue::get(SVT));
530 else
531 ShiftVecAmts.push_back(ConstantInt::get(SVT, Idx));
532 }
533 auto ShiftVec = ConstantVector::get(ShiftVecAmts);
534
535 if (ShiftLeft)
536 return Builder.CreateShl(Vec, ShiftVec);
537
538 if (LogicalShift)
539 return Builder.CreateLShr(Vec, ShiftVec);
540
541 return Builder.CreateAShr(Vec, ShiftVec);
542}
543
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000544static Value *simplifyX86pack(IntrinsicInst &II,
545 InstCombiner::BuilderTy &Builder, bool IsSigned) {
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000546 Value *Arg0 = II.getArgOperand(0);
547 Value *Arg1 = II.getArgOperand(1);
548 Type *ResTy = II.getType();
549
550 // Fast all undef handling.
551 if (isa<UndefValue>(Arg0) && isa<UndefValue>(Arg1))
552 return UndefValue::get(ResTy);
553
554 Type *ArgTy = Arg0->getType();
555 unsigned NumLanes = ResTy->getPrimitiveSizeInBits() / 128;
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000556 unsigned NumSrcElts = ArgTy->getVectorNumElements();
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000557 assert(ResTy->getVectorNumElements() == (2 * NumSrcElts) &&
558 "Unexpected packing types");
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000559
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000560 unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
561 unsigned DstScalarSizeInBits = ResTy->getScalarSizeInBits();
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000562 unsigned SrcScalarSizeInBits = ArgTy->getScalarSizeInBits();
563 assert(SrcScalarSizeInBits == (2 * DstScalarSizeInBits) &&
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000564 "Unexpected packing types");
565
566 // Constant folding.
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000567 if (!isa<Constant>(Arg0) || !isa<Constant>(Arg1))
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000568 return nullptr;
569
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000570 // Clamp Values - signed/unsigned both use signed clamp values, but they
571 // differ on the min/max values.
572 APInt MinValue, MaxValue;
573 if (IsSigned) {
574 // PACKSS: Truncate signed value with signed saturation.
575 // Source values less than dst minint are saturated to minint.
576 // Source values greater than dst maxint are saturated to maxint.
577 MinValue =
578 APInt::getSignedMinValue(DstScalarSizeInBits).sext(SrcScalarSizeInBits);
579 MaxValue =
580 APInt::getSignedMaxValue(DstScalarSizeInBits).sext(SrcScalarSizeInBits);
581 } else {
582 // PACKUS: Truncate signed value with unsigned saturation.
583 // Source values less than zero are saturated to zero.
584 // Source values greater than dst maxuint are saturated to maxuint.
585 MinValue = APInt::getNullValue(SrcScalarSizeInBits);
586 MaxValue = APInt::getLowBitsSet(SrcScalarSizeInBits, DstScalarSizeInBits);
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000587 }
588
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000589 auto *MinC = Constant::getIntegerValue(ArgTy, MinValue);
590 auto *MaxC = Constant::getIntegerValue(ArgTy, MaxValue);
591 Arg0 = Builder.CreateSelect(Builder.CreateICmpSLT(Arg0, MinC), MinC, Arg0);
592 Arg1 = Builder.CreateSelect(Builder.CreateICmpSLT(Arg1, MinC), MinC, Arg1);
593 Arg0 = Builder.CreateSelect(Builder.CreateICmpSGT(Arg0, MaxC), MaxC, Arg0);
594 Arg1 = Builder.CreateSelect(Builder.CreateICmpSGT(Arg1, MaxC), MaxC, Arg1);
595
Simon Pilgrim48a3b542019-04-25 13:51:57 +0000596 // Shuffle clamped args together at the lane level.
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000597 SmallVector<unsigned, 32> PackMask;
598 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
599 for (unsigned Elt = 0; Elt != NumSrcEltsPerLane; ++Elt)
600 PackMask.push_back(Elt + (Lane * NumSrcEltsPerLane));
601 for (unsigned Elt = 0; Elt != NumSrcEltsPerLane; ++Elt)
602 PackMask.push_back(Elt + (Lane * NumSrcEltsPerLane) + NumSrcElts);
603 }
Simon Pilgrim48a3b542019-04-25 13:51:57 +0000604 auto *Shuffle = Builder.CreateShuffleVector(Arg0, Arg1, PackMask);
Simon Pilgrim55f14da2019-04-24 16:53:17 +0000605
Simon Pilgrim48a3b542019-04-25 13:51:57 +0000606 // Truncate to dst size.
607 return Builder.CreateTrunc(Shuffle, ResTy);
Simon Pilgrim6f6b2792017-01-25 14:37:24 +0000608}
609
Mikhail Dvoretckii8393f902018-06-19 10:49:12 +0000610// Replace X86-specific intrinsics with generic floor-ceil where applicable.
611static Value *simplifyX86round(IntrinsicInst &II,
612 InstCombiner::BuilderTy &Builder) {
613 ConstantInt *Arg = nullptr;
614 Intrinsic::ID IntrinsicID = II.getIntrinsicID();
615
616 if (IntrinsicID == Intrinsic::x86_sse41_round_ss ||
617 IntrinsicID == Intrinsic::x86_sse41_round_sd)
618 Arg = dyn_cast<ConstantInt>(II.getArgOperand(2));
619 else if (IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ss ||
620 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_sd)
621 Arg = dyn_cast<ConstantInt>(II.getArgOperand(4));
622 else
623 Arg = dyn_cast<ConstantInt>(II.getArgOperand(1));
624 if (!Arg)
625 return nullptr;
626 unsigned RoundControl = Arg->getZExtValue();
627
628 Arg = nullptr;
629 unsigned SAE = 0;
630 if (IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ps_512 ||
631 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_pd_512)
632 Arg = dyn_cast<ConstantInt>(II.getArgOperand(4));
633 else if (IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ss ||
634 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_sd)
635 Arg = dyn_cast<ConstantInt>(II.getArgOperand(5));
636 else
637 SAE = 4;
638 if (!SAE) {
639 if (!Arg)
640 return nullptr;
641 SAE = Arg->getZExtValue();
642 }
643
644 if (SAE != 4 || (RoundControl != 2 /*ceil*/ && RoundControl != 1 /*floor*/))
645 return nullptr;
646
647 Value *Src, *Dst, *Mask;
648 bool IsScalar = false;
649 if (IntrinsicID == Intrinsic::x86_sse41_round_ss ||
650 IntrinsicID == Intrinsic::x86_sse41_round_sd ||
651 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ss ||
652 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_sd) {
653 IsScalar = true;
654 if (IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ss ||
655 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_sd) {
656 Mask = II.getArgOperand(3);
657 Value *Zero = Constant::getNullValue(Mask->getType());
658 Mask = Builder.CreateAnd(Mask, 1);
659 Mask = Builder.CreateICmp(ICmpInst::ICMP_NE, Mask, Zero);
660 Dst = II.getArgOperand(2);
661 } else
662 Dst = II.getArgOperand(0);
663 Src = Builder.CreateExtractElement(II.getArgOperand(1), (uint64_t)0);
664 } else {
665 Src = II.getArgOperand(0);
666 if (IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ps_128 ||
667 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ps_256 ||
668 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ps_512 ||
669 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_pd_128 ||
670 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_pd_256 ||
671 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_pd_512) {
672 Dst = II.getArgOperand(2);
673 Mask = II.getArgOperand(3);
674 } else {
675 Dst = Src;
676 Mask = ConstantInt::getAllOnesValue(
677 Builder.getIntNTy(Src->getType()->getVectorNumElements()));
678 }
679 }
680
681 Intrinsic::ID ID = (RoundControl == 2) ? Intrinsic::ceil : Intrinsic::floor;
Neil Henning57f5d0a2018-10-08 10:32:33 +0000682 Value *Res = Builder.CreateUnaryIntrinsic(ID, Src, &II);
Mikhail Dvoretckii8393f902018-06-19 10:49:12 +0000683 if (!IsScalar) {
684 if (auto *C = dyn_cast<Constant>(Mask))
685 if (C->isAllOnesValue())
686 return Res;
687 auto *MaskTy = VectorType::get(
688 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
689 Mask = Builder.CreateBitCast(Mask, MaskTy);
690 unsigned Width = Src->getType()->getVectorNumElements();
691 if (MaskTy->getVectorNumElements() > Width) {
692 uint32_t Indices[4];
693 for (unsigned i = 0; i != Width; ++i)
694 Indices[i] = i;
695 Mask = Builder.CreateShuffleVector(Mask, Mask,
696 makeArrayRef(Indices, Width));
697 }
698 return Builder.CreateSelect(Mask, Res, Dst);
699 }
700 if (IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_ss ||
701 IntrinsicID == Intrinsic::x86_avx512_mask_rndscale_sd) {
702 Dst = Builder.CreateExtractElement(Dst, (uint64_t)0);
703 Res = Builder.CreateSelect(Mask, Res, Dst);
704 Dst = II.getArgOperand(0);
705 }
706 return Builder.CreateInsertElement(Dst, Res, (uint64_t)0);
707}
708
Sanjay Patel2aa2dc72018-12-11 16:38:03 +0000709static Value *simplifyX86movmsk(const IntrinsicInst &II,
710 InstCombiner::BuilderTy &Builder) {
Simon Pilgrim91e3ac82016-06-07 08:18:35 +0000711 Value *Arg = II.getArgOperand(0);
712 Type *ResTy = II.getType();
713 Type *ArgTy = Arg->getType();
714
715 // movmsk(undef) -> zero as we must ensure the upper bits are zero.
716 if (isa<UndefValue>(Arg))
717 return Constant::getNullValue(ResTy);
718
719 // We can't easily peek through x86_mmx types.
720 if (!ArgTy->isVectorTy())
721 return nullptr;
722
Simon Pilgrimb4f1bfa2019-04-08 13:17:51 +0000723 // Expand MOVMSK to compare/bitcast/zext:
724 // e.g. PMOVMSKB(v16i8 x):
725 // %cmp = icmp slt <16 x i8> %x, zeroinitializer
726 // %int = bitcast <16 x i1> %cmp to i16
727 // %res = zext i16 %int to i32
728 unsigned NumElts = ArgTy->getVectorNumElements();
729 Type *IntegerVecTy = VectorType::getInteger(cast<VectorType>(ArgTy));
730 Type *IntegerTy = Builder.getIntNTy(NumElts);
Simon Pilgrim91e3ac82016-06-07 08:18:35 +0000731
Simon Pilgrimb4f1bfa2019-04-08 13:17:51 +0000732 Value *Res = Builder.CreateBitCast(Arg, IntegerVecTy);
733 Res = Builder.CreateICmpSLT(Res, Constant::getNullValue(IntegerVecTy));
734 Res = Builder.CreateBitCast(Res, IntegerTy);
735 Res = Builder.CreateZExtOrTrunc(Res, ResTy);
736 return Res;
Simon Pilgrim91e3ac82016-06-07 08:18:35 +0000737}
738
Sanjay Patelbe23a912019-02-01 14:14:47 +0000739static Value *simplifyX86addcarry(const IntrinsicInst &II,
740 InstCombiner::BuilderTy &Builder) {
741 Value *CarryIn = II.getArgOperand(0);
742 Value *Op1 = II.getArgOperand(1);
743 Value *Op2 = II.getArgOperand(2);
744 Type *RetTy = II.getType();
745 Type *OpTy = Op1->getType();
746 assert(RetTy->getStructElementType(0)->isIntegerTy(8) &&
747 RetTy->getStructElementType(1) == OpTy && OpTy == Op2->getType() &&
748 "Unexpected types for x86 addcarry");
749
750 // If carry-in is zero, this is just an unsigned add with overflow.
751 if (match(CarryIn, m_ZeroInt())) {
752 Value *UAdd = Builder.CreateIntrinsic(Intrinsic::uadd_with_overflow, OpTy,
753 { Op1, Op2 });
754 // The types have to be adjusted to match the x86 call types.
755 Value *UAddResult = Builder.CreateExtractValue(UAdd, 0);
756 Value *UAddOV = Builder.CreateZExt(Builder.CreateExtractValue(UAdd, 1),
757 Builder.getInt8Ty());
Sanjay Patelfbcbac72019-02-01 14:37:49 +0000758 Value *Res = UndefValue::get(RetTy);
Sanjay Patelbe23a912019-02-01 14:14:47 +0000759 Res = Builder.CreateInsertValue(Res, UAddOV, 0);
760 return Builder.CreateInsertValue(Res, UAddResult, 1);
761 }
762
763 return nullptr;
764}
765
Sanjay Patel6038d3e2016-01-29 23:27:03 +0000766static Value *simplifyX86insertps(const IntrinsicInst &II,
Sanjay Patelc86867c2015-04-16 17:52:13 +0000767 InstCombiner::BuilderTy &Builder) {
Sanjay Patel03c03f52016-01-28 00:03:16 +0000768 auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2));
769 if (!CInt)
770 return nullptr;
Simon Pilgrim54fcd622015-07-25 20:41:00 +0000771
Sanjay Patel03c03f52016-01-28 00:03:16 +0000772 VectorType *VecTy = cast<VectorType>(II.getType());
773 assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
Sanjay Patelc86867c2015-04-16 17:52:13 +0000774
Sanjay Patel03c03f52016-01-28 00:03:16 +0000775 // The immediate permute control byte looks like this:
776 // [3:0] - zero mask for each 32-bit lane
777 // [5:4] - select one 32-bit destination lane
778 // [7:6] - select one 32-bit source lane
Sanjay Patelc86867c2015-04-16 17:52:13 +0000779
Sanjay Patel03c03f52016-01-28 00:03:16 +0000780 uint8_t Imm = CInt->getZExtValue();
781 uint8_t ZMask = Imm & 0xf;
782 uint8_t DestLane = (Imm >> 4) & 0x3;
783 uint8_t SourceLane = (Imm >> 6) & 0x3;
Sanjay Patelc1d20a32015-04-25 20:55:25 +0000784
Sanjay Patel03c03f52016-01-28 00:03:16 +0000785 ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy);
Sanjay Patelc86867c2015-04-16 17:52:13 +0000786
Sanjay Patel03c03f52016-01-28 00:03:16 +0000787 // If all zero mask bits are set, this was just a weird way to
788 // generate a zero vector.
789 if (ZMask == 0xf)
790 return ZeroVector;
Sanjay Patelc1d20a32015-04-25 20:55:25 +0000791
Sanjay Patel03c03f52016-01-28 00:03:16 +0000792 // Initialize by passing all of the first source bits through.
Craig Topper99d1eab2016-06-12 00:41:19 +0000793 uint32_t ShuffleMask[4] = { 0, 1, 2, 3 };
Sanjay Patelc1d20a32015-04-25 20:55:25 +0000794
Sanjay Patel03c03f52016-01-28 00:03:16 +0000795 // We may replace the second operand with the zero vector.
796 Value *V1 = II.getArgOperand(1);
797
798 if (ZMask) {
799 // If the zero mask is being used with a single input or the zero mask
800 // overrides the destination lane, this is a shuffle with the zero vector.
801 if ((II.getArgOperand(0) == II.getArgOperand(1)) ||
802 (ZMask & (1 << DestLane))) {
803 V1 = ZeroVector;
804 // We may still move 32-bits of the first source vector from one lane
805 // to another.
806 ShuffleMask[DestLane] = SourceLane;
807 // The zero mask may override the previous insert operation.
808 for (unsigned i = 0; i < 4; ++i)
809 if ((ZMask >> i) & 0x1)
810 ShuffleMask[i] = i + 4;
Sanjay Patelc1d20a32015-04-25 20:55:25 +0000811 } else {
Sanjay Patel03c03f52016-01-28 00:03:16 +0000812 // TODO: Model this case as 2 shuffles or a 'logical and' plus shuffle?
813 return nullptr;
Sanjay Patelc1d20a32015-04-25 20:55:25 +0000814 }
Sanjay Patel03c03f52016-01-28 00:03:16 +0000815 } else {
816 // Replace the selected destination lane with the selected source lane.
817 ShuffleMask[DestLane] = SourceLane + 4;
Sanjay Patelc86867c2015-04-16 17:52:13 +0000818 }
Sanjay Patel03c03f52016-01-28 00:03:16 +0000819
820 return Builder.CreateShuffleVector(II.getArgOperand(0), V1, ShuffleMask);
Sanjay Patelc86867c2015-04-16 17:52:13 +0000821}
822
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000823/// Attempt to simplify SSE4A EXTRQ/EXTRQI instructions using constant folding
824/// or conversion to a shuffle vector.
Sanjay Patel6038d3e2016-01-29 23:27:03 +0000825static Value *simplifyX86extrq(IntrinsicInst &II, Value *Op0,
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000826 ConstantInt *CILength, ConstantInt *CIIndex,
827 InstCombiner::BuilderTy &Builder) {
828 auto LowConstantHighUndef = [&](uint64_t Val) {
829 Type *IntTy64 = Type::getInt64Ty(II.getContext());
830 Constant *Args[] = {ConstantInt::get(IntTy64, Val),
831 UndefValue::get(IntTy64)};
832 return ConstantVector::get(Args);
833 };
834
835 // See if we're dealing with constant values.
836 Constant *C0 = dyn_cast<Constant>(Op0);
837 ConstantInt *CI0 =
Andrea Di Biagio8df5b9c2016-09-07 12:03:03 +0000838 C0 ? dyn_cast_or_null<ConstantInt>(C0->getAggregateElement((unsigned)0))
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000839 : nullptr;
840
841 // Attempt to constant fold.
842 if (CILength && CIIndex) {
843 // From AMD documentation: "The bit index and field length are each six
844 // bits in length other bits of the field are ignored."
845 APInt APIndex = CIIndex->getValue().zextOrTrunc(6);
846 APInt APLength = CILength->getValue().zextOrTrunc(6);
847
848 unsigned Index = APIndex.getZExtValue();
849
850 // From AMD documentation: "a value of zero in the field length is
851 // defined as length of 64".
852 unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
853
854 // From AMD documentation: "If the sum of the bit index + length field
855 // is greater than 64, the results are undefined".
856 unsigned End = Index + Length;
857
858 // Note that both field index and field length are 8-bit quantities.
859 // Since variables 'Index' and 'Length' are unsigned values
860 // obtained from zero-extending field index and field length
861 // respectively, their sum should never wrap around.
862 if (End > 64)
863 return UndefValue::get(II.getType());
864
865 // If we are inserting whole bytes, we can convert this to a shuffle.
866 // Lowering can recognize EXTRQI shuffle masks.
867 if ((Length % 8) == 0 && (Index % 8) == 0) {
868 // Convert bit indices to byte indices.
869 Length /= 8;
870 Index /= 8;
871
872 Type *IntTy8 = Type::getInt8Ty(II.getContext());
873 Type *IntTy32 = Type::getInt32Ty(II.getContext());
874 VectorType *ShufTy = VectorType::get(IntTy8, 16);
875
876 SmallVector<Constant *, 16> ShuffleMask;
877 for (int i = 0; i != (int)Length; ++i)
878 ShuffleMask.push_back(
879 Constant::getIntegerValue(IntTy32, APInt(32, i + Index)));
880 for (int i = Length; i != 8; ++i)
881 ShuffleMask.push_back(
882 Constant::getIntegerValue(IntTy32, APInt(32, i + 16)));
883 for (int i = 8; i != 16; ++i)
884 ShuffleMask.push_back(UndefValue::get(IntTy32));
885
886 Value *SV = Builder.CreateShuffleVector(
887 Builder.CreateBitCast(Op0, ShufTy),
888 ConstantAggregateZero::get(ShufTy), ConstantVector::get(ShuffleMask));
889 return Builder.CreateBitCast(SV, II.getType());
890 }
891
892 // Constant Fold - shift Index'th bit to lowest position and mask off
893 // Length bits.
894 if (CI0) {
895 APInt Elt = CI0->getValue();
Craig Topperfc947bc2017-04-18 17:14:21 +0000896 Elt.lshrInPlace(Index);
897 Elt = Elt.zextOrTrunc(Length);
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000898 return LowConstantHighUndef(Elt.getZExtValue());
899 }
900
901 // If we were an EXTRQ call, we'll save registers if we convert to EXTRQI.
902 if (II.getIntrinsicID() == Intrinsic::x86_sse4a_extrq) {
903 Value *Args[] = {Op0, CILength, CIIndex};
Sanjay Patelaf674fb2015-12-14 17:24:23 +0000904 Module *M = II.getModule();
James Y Knight7976eb52019-02-01 20:43:25 +0000905 Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi);
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000906 return Builder.CreateCall(F, Args);
907 }
908 }
909
910 // Constant Fold - extraction from zero is always {zero, undef}.
Craig Topperca2c8762017-07-06 18:39:49 +0000911 if (CI0 && CI0->isZero())
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000912 return LowConstantHighUndef(0);
913
914 return nullptr;
915}
916
917/// Attempt to simplify SSE4A INSERTQ/INSERTQI instructions using constant
918/// folding or conversion to a shuffle vector.
Sanjay Patel6038d3e2016-01-29 23:27:03 +0000919static Value *simplifyX86insertq(IntrinsicInst &II, Value *Op0, Value *Op1,
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000920 APInt APLength, APInt APIndex,
921 InstCombiner::BuilderTy &Builder) {
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000922 // From AMD documentation: "The bit index and field length are each six bits
923 // in length other bits of the field are ignored."
924 APIndex = APIndex.zextOrTrunc(6);
925 APLength = APLength.zextOrTrunc(6);
926
927 // Attempt to constant fold.
928 unsigned Index = APIndex.getZExtValue();
929
930 // From AMD documentation: "a value of zero in the field length is
931 // defined as length of 64".
932 unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
933
934 // From AMD documentation: "If the sum of the bit index + length field
935 // is greater than 64, the results are undefined".
936 unsigned End = Index + Length;
937
938 // Note that both field index and field length are 8-bit quantities.
939 // Since variables 'Index' and 'Length' are unsigned values
940 // obtained from zero-extending field index and field length
941 // respectively, their sum should never wrap around.
942 if (End > 64)
943 return UndefValue::get(II.getType());
944
945 // If we are inserting whole bytes, we can convert this to a shuffle.
946 // Lowering can recognize INSERTQI shuffle masks.
947 if ((Length % 8) == 0 && (Index % 8) == 0) {
948 // Convert bit indices to byte indices.
949 Length /= 8;
950 Index /= 8;
951
952 Type *IntTy8 = Type::getInt8Ty(II.getContext());
953 Type *IntTy32 = Type::getInt32Ty(II.getContext());
954 VectorType *ShufTy = VectorType::get(IntTy8, 16);
955
956 SmallVector<Constant *, 16> ShuffleMask;
957 for (int i = 0; i != (int)Index; ++i)
958 ShuffleMask.push_back(Constant::getIntegerValue(IntTy32, APInt(32, i)));
959 for (int i = 0; i != (int)Length; ++i)
960 ShuffleMask.push_back(
961 Constant::getIntegerValue(IntTy32, APInt(32, i + 16)));
962 for (int i = Index + Length; i != 8; ++i)
963 ShuffleMask.push_back(Constant::getIntegerValue(IntTy32, APInt(32, i)));
964 for (int i = 8; i != 16; ++i)
965 ShuffleMask.push_back(UndefValue::get(IntTy32));
966
967 Value *SV = Builder.CreateShuffleVector(Builder.CreateBitCast(Op0, ShufTy),
968 Builder.CreateBitCast(Op1, ShufTy),
969 ConstantVector::get(ShuffleMask));
970 return Builder.CreateBitCast(SV, II.getType());
971 }
972
973 // See if we're dealing with constant values.
974 Constant *C0 = dyn_cast<Constant>(Op0);
975 Constant *C1 = dyn_cast<Constant>(Op1);
976 ConstantInt *CI00 =
Andrea Di Biagiof3fd3162016-09-07 12:47:53 +0000977 C0 ? dyn_cast_or_null<ConstantInt>(C0->getAggregateElement((unsigned)0))
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000978 : nullptr;
979 ConstantInt *CI10 =
Andrea Di Biagiof3fd3162016-09-07 12:47:53 +0000980 C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)0))
Simon Pilgrim216b1bf2015-10-17 11:40:05 +0000981 : nullptr;
982
983 // Constant Fold - insert bottom Length bits starting at the Index'th bit.
984 if (CI00 && CI10) {
985 APInt V00 = CI00->getValue();
986 APInt V10 = CI10->getValue();
987 APInt Mask = APInt::getLowBitsSet(64, Length).shl(Index);
988 V00 = V00 & ~Mask;
989 V10 = V10.zextOrTrunc(Length).zextOrTrunc(64).shl(Index);
990 APInt Val = V00 | V10;
991 Type *IntTy64 = Type::getInt64Ty(II.getContext());
992 Constant *Args[] = {ConstantInt::get(IntTy64, Val.getZExtValue()),
993 UndefValue::get(IntTy64)};
994 return ConstantVector::get(Args);
995 }
996
997 // If we were an INSERTQ call, we'll save demanded elements if we convert to
998 // INSERTQI.
999 if (II.getIntrinsicID() == Intrinsic::x86_sse4a_insertq) {
1000 Type *IntTy8 = Type::getInt8Ty(II.getContext());
1001 Constant *CILength = ConstantInt::get(IntTy8, Length, false);
1002 Constant *CIIndex = ConstantInt::get(IntTy8, Index, false);
1003
1004 Value *Args[] = {Op0, Op1, CILength, CIIndex};
Sanjay Patelaf674fb2015-12-14 17:24:23 +00001005 Module *M = II.getModule();
James Y Knight7976eb52019-02-01 20:43:25 +00001006 Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00001007 return Builder.CreateCall(F, Args);
1008 }
1009
1010 return nullptr;
1011}
1012
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00001013/// Attempt to convert pshufb* to shufflevector if the mask is constant.
1014static Value *simplifyX86pshufb(const IntrinsicInst &II,
1015 InstCombiner::BuilderTy &Builder) {
Simon Pilgrimbf60cc42016-04-29 21:34:54 +00001016 Constant *V = dyn_cast<Constant>(II.getArgOperand(1));
1017 if (!V)
1018 return nullptr;
1019
Simon Pilgrime5e8c2f2016-05-01 19:26:21 +00001020 auto *VecTy = cast<VectorType>(II.getType());
1021 auto *MaskEltTy = Type::getInt32Ty(II.getContext());
1022 unsigned NumElts = VecTy->getNumElements();
Craig Topper9a63d7a2016-12-11 00:23:50 +00001023 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00001024 "Unexpected number of elements in shuffle mask!");
Simon Pilgrimbf60cc42016-04-29 21:34:54 +00001025
Simon Pilgrime5e8c2f2016-05-01 19:26:21 +00001026 // Construct a shuffle mask from constant integers or UNDEFs.
Craig Topper9a63d7a2016-12-11 00:23:50 +00001027 Constant *Indexes[64] = {nullptr};
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00001028
Simon Pilgrimbf60cc42016-04-29 21:34:54 +00001029 // Each byte in the shuffle control mask forms an index to permute the
1030 // corresponding byte in the destination operand.
1031 for (unsigned I = 0; I < NumElts; ++I) {
1032 Constant *COp = V->getAggregateElement(I);
Simon Pilgrime5e8c2f2016-05-01 19:26:21 +00001033 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
Simon Pilgrimbf60cc42016-04-29 21:34:54 +00001034 return nullptr;
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00001035
Simon Pilgrime5e8c2f2016-05-01 19:26:21 +00001036 if (isa<UndefValue>(COp)) {
1037 Indexes[I] = UndefValue::get(MaskEltTy);
1038 continue;
1039 }
1040
Simon Pilgrimbf60cc42016-04-29 21:34:54 +00001041 int8_t Index = cast<ConstantInt>(COp)->getValue().getZExtValue();
1042
1043 // If the most significant bit (bit[7]) of each byte of the shuffle
1044 // control mask is set, then zero is written in the result byte.
1045 // The zero vector is in the right-hand side of the resulting
1046 // shufflevector.
1047
Simon Pilgrime5e8c2f2016-05-01 19:26:21 +00001048 // The value of each index for the high 128-bit lane is the least
1049 // significant 4 bits of the respective shuffle control byte.
1050 Index = ((Index < 0) ? NumElts : Index & 0x0F) + (I & 0xF0);
1051 Indexes[I] = ConstantInt::get(MaskEltTy, Index);
Simon Pilgrimbf60cc42016-04-29 21:34:54 +00001052 }
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00001053
Simon Pilgrime5e8c2f2016-05-01 19:26:21 +00001054 auto ShuffleMask = ConstantVector::get(makeArrayRef(Indexes, NumElts));
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00001055 auto V1 = II.getArgOperand(0);
Simon Pilgrime5e8c2f2016-05-01 19:26:21 +00001056 auto V2 = Constant::getNullValue(VecTy);
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00001057 return Builder.CreateShuffleVector(V1, V2, ShuffleMask);
1058}
1059
Simon Pilgrim2f6097d2016-04-24 17:23:46 +00001060/// Attempt to convert vpermilvar* to shufflevector if the mask is constant.
1061static Value *simplifyX86vpermilvar(const IntrinsicInst &II,
1062 InstCombiner::BuilderTy &Builder) {
Simon Pilgrim640f9962016-04-30 07:23:30 +00001063 Constant *V = dyn_cast<Constant>(II.getArgOperand(1));
1064 if (!V)
1065 return nullptr;
Simon Pilgrim2f6097d2016-04-24 17:23:46 +00001066
Craig Topper58917f32016-12-11 01:59:36 +00001067 auto *VecTy = cast<VectorType>(II.getType());
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001068 auto *MaskEltTy = Type::getInt32Ty(II.getContext());
Craig Topper58917f32016-12-11 01:59:36 +00001069 unsigned NumElts = VecTy->getVectorNumElements();
1070 bool IsPD = VecTy->getScalarType()->isDoubleTy();
1071 unsigned NumLaneElts = IsPD ? 2 : 4;
1072 assert(NumElts == 16 || NumElts == 8 || NumElts == 4 || NumElts == 2);
Simon Pilgrim2f6097d2016-04-24 17:23:46 +00001073
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001074 // Construct a shuffle mask from constant integers or UNDEFs.
Craig Topper58917f32016-12-11 01:59:36 +00001075 Constant *Indexes[16] = {nullptr};
Simon Pilgrim640f9962016-04-30 07:23:30 +00001076
1077 // The intrinsics only read one or two bits, clear the rest.
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001078 for (unsigned I = 0; I < NumElts; ++I) {
Simon Pilgrim640f9962016-04-30 07:23:30 +00001079 Constant *COp = V->getAggregateElement(I);
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001080 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
Simon Pilgrim640f9962016-04-30 07:23:30 +00001081 return nullptr;
1082
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001083 if (isa<UndefValue>(COp)) {
1084 Indexes[I] = UndefValue::get(MaskEltTy);
1085 continue;
1086 }
1087
1088 APInt Index = cast<ConstantInt>(COp)->getValue();
1089 Index = Index.zextOrTrunc(32).getLoBits(2);
Simon Pilgrim640f9962016-04-30 07:23:30 +00001090
1091 // The PD variants uses bit 1 to select per-lane element index, so
1092 // shift down to convert to generic shuffle mask index.
Craig Topper58917f32016-12-11 01:59:36 +00001093 if (IsPD)
Craig Topperfc947bc2017-04-18 17:14:21 +00001094 Index.lshrInPlace(1);
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001095
1096 // The _256 variants are a bit trickier since the mask bits always index
1097 // into the corresponding 128 half. In order to convert to a generic
1098 // shuffle, we have to make that explicit.
Craig Topper58917f32016-12-11 01:59:36 +00001099 Index += APInt(32, (I / NumLaneElts) * NumLaneElts);
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001100
1101 Indexes[I] = ConstantInt::get(MaskEltTy, Index);
Simon Pilgrim2f6097d2016-04-24 17:23:46 +00001102 }
1103
Simon Pilgrimeeacc402016-05-01 20:22:42 +00001104 auto ShuffleMask = ConstantVector::get(makeArrayRef(Indexes, NumElts));
Simon Pilgrim2f6097d2016-04-24 17:23:46 +00001105 auto V1 = II.getArgOperand(0);
1106 auto V2 = UndefValue::get(V1->getType());
1107 return Builder.CreateShuffleVector(V1, V2, ShuffleMask);
1108}
1109
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00001110/// Attempt to convert vpermd/vpermps to shufflevector if the mask is constant.
1111static Value *simplifyX86vpermv(const IntrinsicInst &II,
1112 InstCombiner::BuilderTy &Builder) {
1113 auto *V = dyn_cast<Constant>(II.getArgOperand(1));
1114 if (!V)
1115 return nullptr;
1116
Simon Pilgrimca140b12016-05-01 20:43:02 +00001117 auto *VecTy = cast<VectorType>(II.getType());
1118 auto *MaskEltTy = Type::getInt32Ty(II.getContext());
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00001119 unsigned Size = VecTy->getNumElements();
Craig Toppere3280452016-12-25 23:58:57 +00001120 assert((Size == 4 || Size == 8 || Size == 16 || Size == 32 || Size == 64) &&
1121 "Unexpected shuffle mask size");
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00001122
Simon Pilgrimca140b12016-05-01 20:43:02 +00001123 // Construct a shuffle mask from constant integers or UNDEFs.
Craig Toppere3280452016-12-25 23:58:57 +00001124 Constant *Indexes[64] = {nullptr};
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00001125
1126 for (unsigned I = 0; I < Size; ++I) {
1127 Constant *COp = V->getAggregateElement(I);
Simon Pilgrimca140b12016-05-01 20:43:02 +00001128 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00001129 return nullptr;
1130
Simon Pilgrimca140b12016-05-01 20:43:02 +00001131 if (isa<UndefValue>(COp)) {
1132 Indexes[I] = UndefValue::get(MaskEltTy);
1133 continue;
1134 }
1135
Craig Toppere3280452016-12-25 23:58:57 +00001136 uint32_t Index = cast<ConstantInt>(COp)->getZExtValue();
1137 Index &= Size - 1;
Simon Pilgrimca140b12016-05-01 20:43:02 +00001138 Indexes[I] = ConstantInt::get(MaskEltTy, Index);
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00001139 }
1140
Simon Pilgrimca140b12016-05-01 20:43:02 +00001141 auto ShuffleMask = ConstantVector::get(makeArrayRef(Indexes, Size));
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00001142 auto V1 = II.getArgOperand(0);
1143 auto V2 = UndefValue::get(VecTy);
1144 return Builder.CreateShuffleVector(V1, V2, ShuffleMask);
1145}
1146
Philip Reames484d07c2019-03-20 03:36:05 +00001147// TODO, Obvious Missing Transforms:
Philip Reames484d07c2019-03-20 03:36:05 +00001148// * Narrow width by halfs excluding zero/undef lanes
Philip Reames7c8647b2019-04-25 01:18:56 +00001149Value *InstCombiner::simplifyMaskedLoad(IntrinsicInst &II) {
Philip Reames2ce01702019-04-23 15:25:14 +00001150 Value *LoadPtr = II.getArgOperand(0);
1151 unsigned Alignment = cast<ConstantInt>(II.getArgOperand(1))->getZExtValue();
1152
David Majnemer666aa942016-07-14 06:58:42 +00001153 // If the mask is all ones or undefs, this is a plain vector load of the 1st
1154 // argument.
Philip Reames2ce01702019-04-23 15:25:14 +00001155 if (maskIsAllOneOrUndef(II.getArgOperand(2)))
James Y Knight14359ef2019-02-01 20:44:24 +00001156 return Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
1157 "unmaskedload");
Philip Reames2ce01702019-04-23 15:25:14 +00001158
1159 // If we can unconditionally load from this address, replace with a
1160 // load/select idiom. TODO: use DT for context sensitive query
1161 if (isDereferenceableAndAlignedPointer(LoadPtr, Alignment,
1162 II.getModule()->getDataLayout(),
1163 &II, nullptr)) {
1164 Value *LI = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
1165 "unmaskedload");
1166 return Builder.CreateSelect(II.getArgOperand(2), LI, II.getArgOperand(3));
Sanjay Patelb695c552016-02-01 17:00:10 +00001167 }
1168
1169 return nullptr;
1170}
1171
Philip Reames484d07c2019-03-20 03:36:05 +00001172// TODO, Obvious Missing Transforms:
Philip Reames484d07c2019-03-20 03:36:05 +00001173// * Single constant active lane -> store
1174// * Narrow width by halfs excluding zero/undef lanes
Philip Reamese4588bb2019-03-20 18:44:58 +00001175Instruction *InstCombiner::simplifyMaskedStore(IntrinsicInst &II) {
Sanjay Patel04f792b2016-02-01 19:39:52 +00001176 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
1177 if (!ConstMask)
1178 return nullptr;
1179
1180 // If the mask is all zeros, this instruction does nothing.
1181 if (ConstMask->isNullValue())
Philip Reamese4588bb2019-03-20 18:44:58 +00001182 return eraseInstFromFunction(II);
Sanjay Patel04f792b2016-02-01 19:39:52 +00001183
1184 // If the mask is all ones, this is a plain vector store of the 1st argument.
1185 if (ConstMask->isAllOnesValue()) {
1186 Value *StorePtr = II.getArgOperand(1);
1187 unsigned Alignment = cast<ConstantInt>(II.getArgOperand(2))->getZExtValue();
1188 return new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);
1189 }
1190
Philip Reamese4588bb2019-03-20 18:44:58 +00001191 // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
1192 APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
1193 APInt UndefElts(DemandedElts.getBitWidth(), 0);
1194 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0),
1195 DemandedElts, UndefElts)) {
1196 II.setOperand(0, V);
1197 return &II;
1198 }
1199
Sanjay Patel04f792b2016-02-01 19:39:52 +00001200 return nullptr;
1201}
1202
Philip Reames484d07c2019-03-20 03:36:05 +00001203// TODO, Obvious Missing Transforms:
1204// * Single constant active lane load -> load
1205// * Dereferenceable address & few lanes -> scalarize speculative load/selects
1206// * Adjacent vector addresses -> masked.load
1207// * Narrow width by halfs excluding zero/undef lanes
Philip Reames60212be2019-03-21 03:23:40 +00001208// * Vector splat address w/known mask -> scalar load
1209// * Vector incrementing address -> vector masked load
Philip Reames7c8647b2019-04-25 01:18:56 +00001210Instruction *InstCombiner::simplifyMaskedGather(IntrinsicInst &II) {
Sanjay Patel103ab7d2016-02-01 22:10:26 +00001211 return nullptr;
1212}
1213
Philip Reames60212be2019-03-21 03:23:40 +00001214// TODO, Obvious Missing Transforms:
1215// * Single constant active lane -> store
1216// * Adjacent vector addresses -> masked.store
1217// * Narrow store width by halfs excluding zero/undef lanes
1218// * Vector splat address w/known mask -> scalar store
1219// * Vector incrementing address -> vector masked store
1220Instruction *InstCombiner::simplifyMaskedScatter(IntrinsicInst &II) {
1221 auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
1222 if (!ConstMask)
1223 return nullptr;
1224
1225 // If the mask is all zeros, a scatter does nothing.
1226 if (ConstMask->isNullValue())
1227 return eraseInstFromFunction(II);
1228
1229 // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
1230 APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
1231 APInt UndefElts(DemandedElts.getBitWidth(), 0);
1232 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0),
1233 DemandedElts, UndefElts)) {
1234 II.setOperand(0, V);
1235 return &II;
1236 }
1237 if (Value *V = SimplifyDemandedVectorElts(II.getOperand(1),
1238 DemandedElts, UndefElts)) {
1239 II.setOperand(1, V);
1240 return &II;
1241 }
1242
1243 return nullptr;
1244}
1245
Piotr Padlewskic63b4922018-07-12 23:55:20 +00001246/// This function transforms launder.invariant.group and strip.invariant.group
1247/// like:
1248/// launder(launder(%x)) -> launder(%x) (the result is not the argument)
1249/// launder(strip(%x)) -> launder(%x)
1250/// strip(strip(%x)) -> strip(%x) (the result is not the argument)
1251/// strip(launder(%x)) -> strip(%x)
1252/// This is legal because it preserves the most recent information about
1253/// the presence or absence of invariant.group.
1254static Instruction *simplifyInvariantGroupIntrinsic(IntrinsicInst &II,
1255 InstCombiner &IC) {
1256 auto *Arg = II.getArgOperand(0);
1257 auto *StrippedArg = Arg->stripPointerCasts();
1258 auto *StrippedInvariantGroupsArg = Arg->stripPointerCastsAndInvariantGroups();
1259 if (StrippedArg == StrippedInvariantGroupsArg)
1260 return nullptr; // No launders/strips to remove.
1261
1262 Value *Result = nullptr;
1263
1264 if (II.getIntrinsicID() == Intrinsic::launder_invariant_group)
1265 Result = IC.Builder.CreateLaunderInvariantGroup(StrippedInvariantGroupsArg);
1266 else if (II.getIntrinsicID() == Intrinsic::strip_invariant_group)
1267 Result = IC.Builder.CreateStripInvariantGroup(StrippedInvariantGroupsArg);
1268 else
1269 llvm_unreachable(
1270 "simplifyInvariantGroupIntrinsic only handles launder and strip");
1271 if (Result->getType()->getPointerAddressSpace() !=
1272 II.getType()->getPointerAddressSpace())
1273 Result = IC.Builder.CreateAddrSpaceCast(Result, II.getType());
1274 if (Result->getType() != II.getType())
1275 Result = IC.Builder.CreateBitCast(Result, II.getType());
1276
1277 return cast<Instruction>(Result);
1278}
1279
Amaury Sechet763c59d2016-08-18 20:43:50 +00001280static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombiner &IC) {
1281 assert((II.getIntrinsicID() == Intrinsic::cttz ||
1282 II.getIntrinsicID() == Intrinsic::ctlz) &&
1283 "Expected cttz or ctlz intrinsic");
David Bolvansky5ba60b22019-04-02 20:13:28 +00001284 bool IsTZ = II.getIntrinsicID() == Intrinsic::cttz;
Sanjay Patel8e3ab172016-08-05 22:42:46 +00001285 Value *Op0 = II.getArgOperand(0);
David Bolvansky5ba60b22019-04-02 20:13:28 +00001286 Value *X;
1287 // ctlz(bitreverse(x)) -> cttz(x)
1288 // cttz(bitreverse(x)) -> ctlz(x)
1289 if (match(Op0, m_BitReverse(m_Value(X)))) {
1290 Intrinsic::ID ID = IsTZ ? Intrinsic::ctlz : Intrinsic::cttz;
1291 Function *F = Intrinsic::getDeclaration(II.getModule(), ID, II.getType());
1292 return CallInst::Create(F, {X, II.getArgOperand(1)});
1293 }
Sanjay Patel8e3ab172016-08-05 22:42:46 +00001294
Craig Topper8205a1a2017-05-24 16:53:07 +00001295 KnownBits Known = IC.computeKnownBits(Op0, 0, &II);
Sanjay Patel8e3ab172016-08-05 22:42:46 +00001296
1297 // Create a mask for bits above (ctlz) or below (cttz) the first known one.
Craig Topper8df66c62017-05-12 17:20:30 +00001298 unsigned PossibleZeros = IsTZ ? Known.countMaxTrailingZeros()
1299 : Known.countMaxLeadingZeros();
1300 unsigned DefiniteZeros = IsTZ ? Known.countMinTrailingZeros()
1301 : Known.countMinLeadingZeros();
Sanjay Patel8e3ab172016-08-05 22:42:46 +00001302
1303 // If all bits above (ctlz) or below (cttz) the first known one are known
1304 // zero, this value is constant.
1305 // FIXME: This should be in InstSimplify because we're replacing an
1306 // instruction with a constant.
Craig Topper9474e9b2017-04-27 04:51:25 +00001307 if (PossibleZeros == DefiniteZeros) {
Craig Topper0799ff92017-06-03 18:50:32 +00001308 auto *C = ConstantInt::get(Op0->getType(), DefiniteZeros);
Amaury Sechet763c59d2016-08-18 20:43:50 +00001309 return IC.replaceInstUsesWith(II, C);
1310 }
1311
1312 // If the input to cttz/ctlz is known to be non-zero,
1313 // then change the 'ZeroIsUndef' parameter to 'true'
1314 // because we know the zero behavior can't affect the result.
Craig Topper73ba1c82017-06-07 07:40:37 +00001315 if (!Known.One.isNullValue() ||
Craig Topperd45185f2017-05-26 18:23:57 +00001316 isKnownNonZero(Op0, IC.getDataLayout(), 0, &IC.getAssumptionCache(), &II,
1317 &IC.getDominatorTree())) {
Amaury Sechet763c59d2016-08-18 20:43:50 +00001318 if (!match(II.getArgOperand(1), m_One())) {
Craig Topperbb4069e2017-07-07 23:16:26 +00001319 II.setOperand(1, IC.Builder.getTrue());
Amaury Sechet763c59d2016-08-18 20:43:50 +00001320 return &II;
1321 }
1322 }
Sanjay Patel8e3ab172016-08-05 22:42:46 +00001323
Craig Topper5b173f22017-06-21 16:32:35 +00001324 // Add range metadata since known bits can't completely reflect what we know.
1325 // TODO: Handle splat vectors.
1326 auto *IT = dyn_cast<IntegerType>(Op0->getType());
1327 if (IT && IT->getBitWidth() != 1 && !II.getMetadata(LLVMContext::MD_range)) {
1328 Metadata *LowAndHigh[] = {
1329 ConstantAsMetadata::get(ConstantInt::get(IT, DefiniteZeros)),
1330 ConstantAsMetadata::get(ConstantInt::get(IT, PossibleZeros + 1))};
1331 II.setMetadata(LLVMContext::MD_range,
1332 MDNode::get(II.getContext(), LowAndHigh));
1333 return &II;
1334 }
1335
1336 return nullptr;
1337}
1338
1339static Instruction *foldCtpop(IntrinsicInst &II, InstCombiner &IC) {
1340 assert(II.getIntrinsicID() == Intrinsic::ctpop &&
1341 "Expected ctpop intrinsic");
1342 Value *Op0 = II.getArgOperand(0);
David Bolvansky937720e2019-04-03 08:08:44 +00001343 Value *X;
1344 // ctpop(bitreverse(x)) -> ctpop(x)
1345 // ctpop(bswap(x)) -> ctpop(x)
1346 if (match(Op0, m_BitReverse(m_Value(X))) || match(Op0, m_BSwap(m_Value(X)))) {
1347 II.setOperand(0, X);
1348 return &II;
1349 }
1350
Craig Topper5b173f22017-06-21 16:32:35 +00001351 // FIXME: Try to simplify vectors of integers.
1352 auto *IT = dyn_cast<IntegerType>(Op0->getType());
1353 if (!IT)
1354 return nullptr;
1355
1356 unsigned BitWidth = IT->getBitWidth();
1357 KnownBits Known(BitWidth);
1358 IC.computeKnownBits(Op0, Known, 0, &II);
1359
1360 unsigned MinCount = Known.countMinPopulation();
1361 unsigned MaxCount = Known.countMaxPopulation();
1362
1363 // Add range metadata since known bits can't completely reflect what we know.
1364 if (IT->getBitWidth() != 1 && !II.getMetadata(LLVMContext::MD_range)) {
1365 Metadata *LowAndHigh[] = {
1366 ConstantAsMetadata::get(ConstantInt::get(IT, MinCount)),
1367 ConstantAsMetadata::get(ConstantInt::get(IT, MaxCount + 1))};
1368 II.setMetadata(LLVMContext::MD_range,
1369 MDNode::get(II.getContext(), LowAndHigh));
1370 return &II;
1371 }
1372
Sanjay Patel8e3ab172016-08-05 22:42:46 +00001373 return nullptr;
1374}
1375
Sanjay Patel1ace9932016-02-26 21:04:14 +00001376// TODO: If the x86 backend knew how to convert a bool vector mask back to an
1377// XMM register mask efficiently, we could transform all x86 masked intrinsics
1378// to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
Sanjay Patel98a71502016-02-29 23:16:48 +00001379static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
1380 Value *Ptr = II.getOperand(0);
1381 Value *Mask = II.getOperand(1);
Sanjay Patel5e5056d2016-04-12 23:16:23 +00001382 Constant *ZeroVec = Constant::getNullValue(II.getType());
Sanjay Patel98a71502016-02-29 23:16:48 +00001383
1384 // Special case a zero mask since that's not a ConstantDataVector.
Sanjay Patel5e5056d2016-04-12 23:16:23 +00001385 // This masked load instruction creates a zero vector.
Sanjay Patel98a71502016-02-29 23:16:48 +00001386 if (isa<ConstantAggregateZero>(Mask))
Sanjay Patel5e5056d2016-04-12 23:16:23 +00001387 return IC.replaceInstUsesWith(II, ZeroVec);
Sanjay Patel98a71502016-02-29 23:16:48 +00001388
1389 auto *ConstMask = dyn_cast<ConstantDataVector>(Mask);
1390 if (!ConstMask)
1391 return nullptr;
1392
1393 // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic
1394 // to allow target-independent optimizations.
1395
1396 // First, cast the x86 intrinsic scalar pointer to a vector pointer to match
1397 // the LLVM intrinsic definition for the pointer argument.
1398 unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
1399 PointerType *VecPtrTy = PointerType::get(II.getType(), AddrSpace);
Craig Topperbb4069e2017-07-07 23:16:26 +00001400 Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
Sanjay Patel98a71502016-02-29 23:16:48 +00001401
1402 // Second, convert the x86 XMM integer vector mask to a vector of bools based
1403 // on each element's most significant bit (the sign bit).
1404 Constant *BoolMask = getNegativeIsTrueBoolVec(ConstMask);
1405
Sanjay Patel5e5056d2016-04-12 23:16:23 +00001406 // The pass-through vector for an x86 masked load is a zero vector.
1407 CallInst *NewMaskedLoad =
Craig Topperbb4069e2017-07-07 23:16:26 +00001408 IC.Builder.CreateMaskedLoad(PtrCast, 1, BoolMask, ZeroVec);
Sanjay Patel98a71502016-02-29 23:16:48 +00001409 return IC.replaceInstUsesWith(II, NewMaskedLoad);
1410}
1411
1412// TODO: If the x86 backend knew how to convert a bool vector mask back to an
1413// XMM register mask efficiently, we could transform all x86 masked intrinsics
1414// to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
Sanjay Patel1ace9932016-02-26 21:04:14 +00001415static bool simplifyX86MaskedStore(IntrinsicInst &II, InstCombiner &IC) {
1416 Value *Ptr = II.getOperand(0);
1417 Value *Mask = II.getOperand(1);
1418 Value *Vec = II.getOperand(2);
1419
1420 // Special case a zero mask since that's not a ConstantDataVector:
1421 // this masked store instruction does nothing.
1422 if (isa<ConstantAggregateZero>(Mask)) {
1423 IC.eraseInstFromFunction(II);
1424 return true;
1425 }
1426
Sanjay Patelc4acbae2016-03-12 15:16:59 +00001427 // The SSE2 version is too weird (eg, unaligned but non-temporal) to do
1428 // anything else at this level.
1429 if (II.getIntrinsicID() == Intrinsic::x86_sse2_maskmov_dqu)
1430 return false;
1431
Sanjay Patel1ace9932016-02-26 21:04:14 +00001432 auto *ConstMask = dyn_cast<ConstantDataVector>(Mask);
1433 if (!ConstMask)
1434 return false;
1435
1436 // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic
1437 // to allow target-independent optimizations.
1438
1439 // First, cast the x86 intrinsic scalar pointer to a vector pointer to match
1440 // the LLVM intrinsic definition for the pointer argument.
1441 unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
1442 PointerType *VecPtrTy = PointerType::get(Vec->getType(), AddrSpace);
Craig Topperbb4069e2017-07-07 23:16:26 +00001443 Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
Sanjay Patel1ace9932016-02-26 21:04:14 +00001444
1445 // Second, convert the x86 XMM integer vector mask to a vector of bools based
1446 // on each element's most significant bit (the sign bit).
1447 Constant *BoolMask = getNegativeIsTrueBoolVec(ConstMask);
1448
Craig Topperbb4069e2017-07-07 23:16:26 +00001449 IC.Builder.CreateMaskedStore(Vec, PtrCast, 1, BoolMask);
Sanjay Patel1ace9932016-02-26 21:04:14 +00001450
1451 // 'Replace uses' doesn't work for stores. Erase the original masked store.
1452 IC.eraseInstFromFunction(II);
1453 return true;
1454}
1455
Matt Arsenaultcdb468c2017-02-27 23:08:49 +00001456// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
1457//
1458// A single NaN input is folded to minnum, so we rely on that folding for
1459// handling NaNs.
1460static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
1461 const APFloat &Src2) {
1462 APFloat Max3 = maxnum(maxnum(Src0, Src1), Src2);
1463
1464 APFloat::cmpResult Cmp0 = Max3.compare(Src0);
1465 assert(Cmp0 != APFloat::cmpUnordered && "nans handled separately");
1466 if (Cmp0 == APFloat::cmpEqual)
1467 return maxnum(Src1, Src2);
1468
1469 APFloat::cmpResult Cmp1 = Max3.compare(Src1);
1470 assert(Cmp1 != APFloat::cmpUnordered && "nans handled separately");
1471 if (Cmp1 == APFloat::cmpEqual)
1472 return maxnum(Src0, Src2);
1473
1474 return maxnum(Src0, Src1);
1475}
1476
Alexandros Lamprineas52457d32018-05-30 14:38:50 +00001477/// Convert a table lookup to shufflevector if the mask is constant.
1478/// This could benefit tbl1 if the mask is { 7,6,5,4,3,2,1,0 }, in
1479/// which case we could lower the shufflevector with rev64 instructions
1480/// as it's actually a byte reverse.
1481static Value *simplifyNeonTbl1(const IntrinsicInst &II,
1482 InstCombiner::BuilderTy &Builder) {
1483 // Bail out if the mask is not a constant.
1484 auto *C = dyn_cast<Constant>(II.getArgOperand(1));
1485 if (!C)
1486 return nullptr;
1487
1488 auto *VecTy = cast<VectorType>(II.getType());
1489 unsigned NumElts = VecTy->getNumElements();
1490
1491 // Only perform this transformation for <8 x i8> vector types.
1492 if (!VecTy->getElementType()->isIntegerTy(8) || NumElts != 8)
1493 return nullptr;
1494
1495 uint32_t Indexes[8];
1496
1497 for (unsigned I = 0; I < NumElts; ++I) {
1498 Constant *COp = C->getAggregateElement(I);
1499
1500 if (!COp || !isa<ConstantInt>(COp))
1501 return nullptr;
1502
1503 Indexes[I] = cast<ConstantInt>(COp)->getLimitedValue();
1504
1505 // Make sure the mask indices are in range.
1506 if (Indexes[I] >= NumElts)
1507 return nullptr;
1508 }
1509
1510 auto *ShuffleMask = ConstantDataVector::get(II.getContext(),
1511 makeArrayRef(Indexes));
1512 auto *V1 = II.getArgOperand(0);
1513 auto *V2 = Constant::getNullValue(V1->getType());
1514 return Builder.CreateShuffleVector(V1, V2, ShuffleMask);
1515}
1516
Alexandros Lamprineas61f0ba12018-05-31 12:19:18 +00001517/// Convert a vector load intrinsic into a simple llvm load instruction.
1518/// This is beneficial when the underlying object being addressed comes
1519/// from a constant, since we get constant-folding for free.
1520static Value *simplifyNeonVld1(const IntrinsicInst &II,
1521 unsigned MemAlign,
1522 InstCombiner::BuilderTy &Builder) {
1523 auto *IntrAlign = dyn_cast<ConstantInt>(II.getArgOperand(1));
1524
1525 if (!IntrAlign)
1526 return nullptr;
1527
1528 unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign ?
1529 MemAlign : IntrAlign->getLimitedValue();
1530
1531 if (!isPowerOf2_32(Alignment))
1532 return nullptr;
1533
1534 auto *BCastInst = Builder.CreateBitCast(II.getArgOperand(0),
1535 PointerType::get(II.getType(), 0));
James Y Knight14359ef2019-02-01 20:44:24 +00001536 return Builder.CreateAlignedLoad(II.getType(), BCastInst, Alignment);
Alexandros Lamprineas61f0ba12018-05-31 12:19:18 +00001537}
1538
Arnaud A. de Grandmaison333ef382016-05-10 09:24:49 +00001539// Returns true iff the 2 intrinsics have the same operands, limiting the
1540// comparison to the first NumOperands.
1541static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E,
1542 unsigned NumOperands) {
1543 assert(I.getNumArgOperands() >= NumOperands && "Not enough operands");
1544 assert(E.getNumArgOperands() >= NumOperands && "Not enough operands");
1545 for (unsigned i = 0; i < NumOperands; i++)
1546 if (I.getArgOperand(i) != E.getArgOperand(i))
1547 return false;
1548 return true;
1549}
1550
1551// Remove trivially empty start/end intrinsic ranges, i.e. a start
1552// immediately followed by an end (ignoring debuginfo or other
1553// start/end intrinsics in between). As this handles only the most trivial
1554// cases, tracking the nesting level is not needed:
1555//
1556// call @llvm.foo.start(i1 0) ; &I
1557// call @llvm.foo.start(i1 0)
1558// call @llvm.foo.end(i1 0) ; This one will not be skipped: it will be removed
1559// call @llvm.foo.end(i1 0)
1560static bool removeTriviallyEmptyRange(IntrinsicInst &I, unsigned StartID,
1561 unsigned EndID, InstCombiner &IC) {
1562 assert(I.getIntrinsicID() == StartID &&
1563 "Start intrinsic does not have expected ID");
1564 BasicBlock::iterator BI(I), BE(I.getParent()->end());
1565 for (++BI; BI != BE; ++BI) {
1566 if (auto *E = dyn_cast<IntrinsicInst>(BI)) {
1567 if (isa<DbgInfoIntrinsic>(E) || E->getIntrinsicID() == StartID)
1568 continue;
1569 if (E->getIntrinsicID() == EndID &&
1570 haveSameOperands(I, *E, E->getNumArgOperands())) {
1571 IC.eraseInstFromFunction(*E);
1572 IC.eraseInstFromFunction(I);
1573 return true;
1574 }
1575 }
1576 break;
1577 }
1578
1579 return false;
1580}
1581
Justin Lebar698c31b2017-01-27 00:58:58 +00001582// Convert NVVM intrinsics to target-generic LLVM code where possible.
1583static Instruction *SimplifyNVVMIntrinsic(IntrinsicInst *II, InstCombiner &IC) {
1584 // Each NVVM intrinsic we can simplify can be replaced with one of:
1585 //
1586 // * an LLVM intrinsic,
1587 // * an LLVM cast operation,
1588 // * an LLVM binary operation, or
1589 // * ad-hoc LLVM IR for the particular operation.
1590
1591 // Some transformations are only valid when the module's
1592 // flush-denormals-to-zero (ftz) setting is true/false, whereas other
1593 // transformations are valid regardless of the module's ftz setting.
1594 enum FtzRequirementTy {
1595 FTZ_Any, // Any ftz setting is ok.
1596 FTZ_MustBeOn, // Transformation is valid only if ftz is on.
1597 FTZ_MustBeOff, // Transformation is valid only if ftz is off.
1598 };
1599 // Classes of NVVM intrinsics that can't be replaced one-to-one with a
1600 // target-generic intrinsic, cast op, or binary op but that we can nonetheless
1601 // simplify.
1602 enum SpecialCase {
1603 SPC_Reciprocal,
1604 };
1605
1606 // SimplifyAction is a poor-man's variant (plus an additional flag) that
1607 // represents how to replace an NVVM intrinsic with target-generic LLVM IR.
1608 struct SimplifyAction {
1609 // Invariant: At most one of these Optionals has a value.
1610 Optional<Intrinsic::ID> IID;
1611 Optional<Instruction::CastOps> CastOp;
1612 Optional<Instruction::BinaryOps> BinaryOp;
1613 Optional<SpecialCase> Special;
1614
1615 FtzRequirementTy FtzRequirement = FTZ_Any;
1616
1617 SimplifyAction() = default;
1618
1619 SimplifyAction(Intrinsic::ID IID, FtzRequirementTy FtzReq)
1620 : IID(IID), FtzRequirement(FtzReq) {}
1621
1622 // Cast operations don't have anything to do with FTZ, so we skip that
1623 // argument.
1624 SimplifyAction(Instruction::CastOps CastOp) : CastOp(CastOp) {}
1625
1626 SimplifyAction(Instruction::BinaryOps BinaryOp, FtzRequirementTy FtzReq)
1627 : BinaryOp(BinaryOp), FtzRequirement(FtzReq) {}
1628
1629 SimplifyAction(SpecialCase Special, FtzRequirementTy FtzReq)
1630 : Special(Special), FtzRequirement(FtzReq) {}
1631 };
1632
1633 // Try to generate a SimplifyAction describing how to replace our
1634 // IntrinsicInstr with target-generic LLVM IR.
1635 const SimplifyAction Action = [II]() -> SimplifyAction {
1636 switch (II->getIntrinsicID()) {
Justin Lebar698c31b2017-01-27 00:58:58 +00001637 // NVVM intrinsics that map directly to LLVM intrinsics.
1638 case Intrinsic::nvvm_ceil_d:
1639 return {Intrinsic::ceil, FTZ_Any};
1640 case Intrinsic::nvvm_ceil_f:
1641 return {Intrinsic::ceil, FTZ_MustBeOff};
1642 case Intrinsic::nvvm_ceil_ftz_f:
1643 return {Intrinsic::ceil, FTZ_MustBeOn};
1644 case Intrinsic::nvvm_fabs_d:
1645 return {Intrinsic::fabs, FTZ_Any};
1646 case Intrinsic::nvvm_fabs_f:
1647 return {Intrinsic::fabs, FTZ_MustBeOff};
1648 case Intrinsic::nvvm_fabs_ftz_f:
1649 return {Intrinsic::fabs, FTZ_MustBeOn};
1650 case Intrinsic::nvvm_floor_d:
1651 return {Intrinsic::floor, FTZ_Any};
1652 case Intrinsic::nvvm_floor_f:
1653 return {Intrinsic::floor, FTZ_MustBeOff};
1654 case Intrinsic::nvvm_floor_ftz_f:
1655 return {Intrinsic::floor, FTZ_MustBeOn};
1656 case Intrinsic::nvvm_fma_rn_d:
1657 return {Intrinsic::fma, FTZ_Any};
1658 case Intrinsic::nvvm_fma_rn_f:
1659 return {Intrinsic::fma, FTZ_MustBeOff};
1660 case Intrinsic::nvvm_fma_rn_ftz_f:
1661 return {Intrinsic::fma, FTZ_MustBeOn};
1662 case Intrinsic::nvvm_fmax_d:
1663 return {Intrinsic::maxnum, FTZ_Any};
1664 case Intrinsic::nvvm_fmax_f:
1665 return {Intrinsic::maxnum, FTZ_MustBeOff};
1666 case Intrinsic::nvvm_fmax_ftz_f:
1667 return {Intrinsic::maxnum, FTZ_MustBeOn};
1668 case Intrinsic::nvvm_fmin_d:
1669 return {Intrinsic::minnum, FTZ_Any};
1670 case Intrinsic::nvvm_fmin_f:
1671 return {Intrinsic::minnum, FTZ_MustBeOff};
1672 case Intrinsic::nvvm_fmin_ftz_f:
1673 return {Intrinsic::minnum, FTZ_MustBeOn};
1674 case Intrinsic::nvvm_round_d:
1675 return {Intrinsic::round, FTZ_Any};
1676 case Intrinsic::nvvm_round_f:
1677 return {Intrinsic::round, FTZ_MustBeOff};
1678 case Intrinsic::nvvm_round_ftz_f:
1679 return {Intrinsic::round, FTZ_MustBeOn};
1680 case Intrinsic::nvvm_sqrt_rn_d:
1681 return {Intrinsic::sqrt, FTZ_Any};
1682 case Intrinsic::nvvm_sqrt_f:
1683 // nvvm_sqrt_f is a special case. For most intrinsics, foo_ftz_f is the
1684 // ftz version, and foo_f is the non-ftz version. But nvvm_sqrt_f adopts
1685 // the ftz-ness of the surrounding code. sqrt_rn_f and sqrt_rn_ftz_f are
1686 // the versions with explicit ftz-ness.
1687 return {Intrinsic::sqrt, FTZ_Any};
1688 case Intrinsic::nvvm_sqrt_rn_f:
1689 return {Intrinsic::sqrt, FTZ_MustBeOff};
1690 case Intrinsic::nvvm_sqrt_rn_ftz_f:
1691 return {Intrinsic::sqrt, FTZ_MustBeOn};
1692 case Intrinsic::nvvm_trunc_d:
1693 return {Intrinsic::trunc, FTZ_Any};
1694 case Intrinsic::nvvm_trunc_f:
1695 return {Intrinsic::trunc, FTZ_MustBeOff};
1696 case Intrinsic::nvvm_trunc_ftz_f:
1697 return {Intrinsic::trunc, FTZ_MustBeOn};
1698
1699 // NVVM intrinsics that map to LLVM cast operations.
1700 //
1701 // Note that llvm's target-generic conversion operators correspond to the rz
1702 // (round to zero) versions of the nvvm conversion intrinsics, even though
1703 // most everything else here uses the rn (round to nearest even) nvvm ops.
1704 case Intrinsic::nvvm_d2i_rz:
1705 case Intrinsic::nvvm_f2i_rz:
1706 case Intrinsic::nvvm_d2ll_rz:
1707 case Intrinsic::nvvm_f2ll_rz:
1708 return {Instruction::FPToSI};
1709 case Intrinsic::nvvm_d2ui_rz:
1710 case Intrinsic::nvvm_f2ui_rz:
1711 case Intrinsic::nvvm_d2ull_rz:
1712 case Intrinsic::nvvm_f2ull_rz:
1713 return {Instruction::FPToUI};
1714 case Intrinsic::nvvm_i2d_rz:
1715 case Intrinsic::nvvm_i2f_rz:
1716 case Intrinsic::nvvm_ll2d_rz:
1717 case Intrinsic::nvvm_ll2f_rz:
1718 return {Instruction::SIToFP};
1719 case Intrinsic::nvvm_ui2d_rz:
1720 case Intrinsic::nvvm_ui2f_rz:
1721 case Intrinsic::nvvm_ull2d_rz:
1722 case Intrinsic::nvvm_ull2f_rz:
1723 return {Instruction::UIToFP};
1724
1725 // NVVM intrinsics that map to LLVM binary ops.
1726 case Intrinsic::nvvm_add_rn_d:
1727 return {Instruction::FAdd, FTZ_Any};
1728 case Intrinsic::nvvm_add_rn_f:
1729 return {Instruction::FAdd, FTZ_MustBeOff};
1730 case Intrinsic::nvvm_add_rn_ftz_f:
1731 return {Instruction::FAdd, FTZ_MustBeOn};
1732 case Intrinsic::nvvm_mul_rn_d:
1733 return {Instruction::FMul, FTZ_Any};
1734 case Intrinsic::nvvm_mul_rn_f:
1735 return {Instruction::FMul, FTZ_MustBeOff};
1736 case Intrinsic::nvvm_mul_rn_ftz_f:
1737 return {Instruction::FMul, FTZ_MustBeOn};
1738 case Intrinsic::nvvm_div_rn_d:
1739 return {Instruction::FDiv, FTZ_Any};
1740 case Intrinsic::nvvm_div_rn_f:
1741 return {Instruction::FDiv, FTZ_MustBeOff};
1742 case Intrinsic::nvvm_div_rn_ftz_f:
1743 return {Instruction::FDiv, FTZ_MustBeOn};
1744
1745 // The remainder of cases are NVVM intrinsics that map to LLVM idioms, but
1746 // need special handling.
1747 //
Hiroshi Inoue0ca79dc2017-07-11 06:04:59 +00001748 // We seem to be missing intrinsics for rcp.approx.{ftz.}f32, which is just
Justin Lebar698c31b2017-01-27 00:58:58 +00001749 // as well.
1750 case Intrinsic::nvvm_rcp_rn_d:
1751 return {SPC_Reciprocal, FTZ_Any};
1752 case Intrinsic::nvvm_rcp_rn_f:
1753 return {SPC_Reciprocal, FTZ_MustBeOff};
1754 case Intrinsic::nvvm_rcp_rn_ftz_f:
1755 return {SPC_Reciprocal, FTZ_MustBeOn};
1756
1757 // We do not currently simplify intrinsics that give an approximate answer.
1758 // These include:
1759 //
1760 // - nvvm_cos_approx_{f,ftz_f}
1761 // - nvvm_ex2_approx_{d,f,ftz_f}
1762 // - nvvm_lg2_approx_{d,f,ftz_f}
1763 // - nvvm_sin_approx_{f,ftz_f}
1764 // - nvvm_sqrt_approx_{f,ftz_f}
1765 // - nvvm_rsqrt_approx_{d,f,ftz_f}
1766 // - nvvm_div_approx_{ftz_d,ftz_f,f}
1767 // - nvvm_rcp_approx_ftz_d
1768 //
1769 // Ideally we'd encode them as e.g. "fast call @llvm.cos", where "fast"
1770 // means that fastmath is enabled in the intrinsic. Unfortunately only
1771 // binary operators (currently) have a fastmath bit in SelectionDAG, so this
1772 // information gets lost and we can't select on it.
1773 //
1774 // TODO: div and rcp are lowered to a binary op, so these we could in theory
1775 // lower them to "fast fdiv".
1776
1777 default:
1778 return {};
1779 }
1780 }();
1781
1782 // If Action.FtzRequirementTy is not satisfied by the module's ftz state, we
1783 // can bail out now. (Notice that in the case that IID is not an NVVM
1784 // intrinsic, we don't have to look up any module metadata, as
1785 // FtzRequirementTy will be FTZ_Any.)
1786 if (Action.FtzRequirement != FTZ_Any) {
1787 bool FtzEnabled =
1788 II->getFunction()->getFnAttribute("nvptx-f32ftz").getValueAsString() ==
1789 "true";
1790
1791 if (FtzEnabled != (Action.FtzRequirement == FTZ_MustBeOn))
1792 return nullptr;
1793 }
1794
1795 // Simplify to target-generic intrinsic.
1796 if (Action.IID) {
1797 SmallVector<Value *, 4> Args(II->arg_operands());
1798 // All the target-generic intrinsics currently of interest to us have one
1799 // type argument, equal to that of the nvvm intrinsic's argument.
Justin Lebare3ac0fb2017-01-27 01:49:39 +00001800 Type *Tys[] = {II->getArgOperand(0)->getType()};
Justin Lebar698c31b2017-01-27 00:58:58 +00001801 return CallInst::Create(
1802 Intrinsic::getDeclaration(II->getModule(), *Action.IID, Tys), Args);
1803 }
1804
1805 // Simplify to target-generic binary op.
1806 if (Action.BinaryOp)
1807 return BinaryOperator::Create(*Action.BinaryOp, II->getArgOperand(0),
1808 II->getArgOperand(1), II->getName());
1809
1810 // Simplify to target-generic cast op.
1811 if (Action.CastOp)
1812 return CastInst::Create(*Action.CastOp, II->getArgOperand(0), II->getType(),
1813 II->getName());
1814
1815 // All that's left are the special cases.
1816 if (!Action.Special)
1817 return nullptr;
1818
1819 switch (*Action.Special) {
1820 case SPC_Reciprocal:
1821 // Simplify reciprocal.
1822 return BinaryOperator::Create(
1823 Instruction::FDiv, ConstantFP::get(II->getArgOperand(0)->getType(), 1),
1824 II->getArgOperand(0), II->getName());
1825 }
Justin Lebar25ebe2d2017-01-27 02:04:07 +00001826 llvm_unreachable("All SpecialCase enumerators should be handled in switch.");
Justin Lebar698c31b2017-01-27 00:58:58 +00001827}
1828
Arnaud A. de Grandmaison333ef382016-05-10 09:24:49 +00001829Instruction *InstCombiner::visitVAStartInst(VAStartInst &I) {
1830 removeTriviallyEmptyRange(I, Intrinsic::vastart, Intrinsic::vaend, *this);
1831 return nullptr;
1832}
1833
1834Instruction *InstCombiner::visitVACopyInst(VACopyInst &I) {
1835 removeTriviallyEmptyRange(I, Intrinsic::vacopy, Intrinsic::vaend, *this);
1836 return nullptr;
1837}
1838
Sanjay Patel790af912018-11-26 22:00:41 +00001839static Instruction *canonicalizeConstantArg0ToArg1(CallInst &Call) {
1840 assert(Call.getNumArgOperands() > 1 && "Need at least 2 args to swap");
1841 Value *Arg0 = Call.getArgOperand(0), *Arg1 = Call.getArgOperand(1);
1842 if (isa<Constant>(Arg0) && !isa<Constant>(Arg1)) {
1843 Call.setArgOperand(0, Arg1);
1844 Call.setArgOperand(1, Arg0);
1845 return &Call;
1846 }
1847 return nullptr;
1848}
1849
Nikita Popov884feb12019-03-06 18:30:00 +00001850Instruction *InstCombiner::foldIntrinsicWithOverflowCommon(IntrinsicInst *II) {
1851 OverflowCheckFlavor OCF =
1852 IntrinsicIDToOverflowCheckFlavor(II->getIntrinsicID());
1853 assert(OCF != OCF_INVALID && "unexpected!");
1854
1855 Value *OperationResult = nullptr;
1856 Constant *OverflowResult = nullptr;
1857 if (OptimizeOverflowCheck(OCF, II->getArgOperand(0), II->getArgOperand(1),
1858 *II, OperationResult, OverflowResult))
1859 return CreateOverflowTuple(II, OperationResult, OverflowResult);
1860 return nullptr;
1861}
1862
Sanjay Patelcd4377c2016-01-20 22:24:38 +00001863/// CallInst simplification. This mostly only handles folding of intrinsic
Craig Topperc1892ec2019-01-31 17:23:29 +00001864/// instructions. For normal calls, it allows visitCallBase to do the heavy
Sanjay Patelcd4377c2016-01-20 22:24:38 +00001865/// lifting.
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001866Instruction *InstCombiner::visitCallInst(CallInst &CI) {
Philip Reames7a6db4f2017-12-27 00:16:12 +00001867 if (Value *V = SimplifyCall(&CI, SQ.getWithInstruction(&CI)))
Sanjay Patel4b198802016-02-01 22:23:39 +00001868 return replaceInstUsesWith(CI, V);
David Majnemer15032582015-05-22 03:56:46 +00001869
Justin Bogner99798402016-08-05 01:06:44 +00001870 if (isFreeCall(&CI, &TLI))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001871 return visitFree(CI);
1872
1873 // If the caller function is nounwind, mark the call as nounwind, even if the
1874 // callee isn't.
Sanjay Patel5a470952016-08-11 15:16:06 +00001875 if (CI.getFunction()->doesNotThrow() && !CI.doesNotThrow()) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001876 CI.setDoesNotThrow();
1877 return &CI;
1878 }
Jim Grosbach7815f562012-02-03 00:07:04 +00001879
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001880 IntrinsicInst *II = dyn_cast<IntrinsicInst>(&CI);
Craig Topperc1892ec2019-01-31 17:23:29 +00001881 if (!II) return visitCallBase(CI);
Gabor Greif589a0b92010-06-24 12:58:35 +00001882
Craig Topper784929d2019-02-08 20:48:56 +00001883 // Intrinsics cannot occur in an invoke or a callbr, so handle them here
1884 // instead of in visitCallBase.
Daniel Neilson8f30ec62018-05-11 14:30:02 +00001885 if (auto *MI = dyn_cast<AnyMemIntrinsic>(II)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001886 bool Changed = false;
1887
1888 // memmove/cpy/set of zero bytes is a noop.
1889 if (Constant *NumBytes = dyn_cast<Constant>(MI->getLength())) {
Chris Lattnerc663a672010-10-01 05:51:02 +00001890 if (NumBytes->isNullValue())
Sanjay Patel4b198802016-02-01 22:23:39 +00001891 return eraseInstFromFunction(CI);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001892
1893 if (ConstantInt *CI = dyn_cast<ConstantInt>(NumBytes))
1894 if (CI->getZExtValue() == 1) {
1895 // Replace the instruction with just byte operations. We would
1896 // transform other cases to loads/stores, but we don't know if
1897 // alignment is sufficient.
1898 }
1899 }
Jim Grosbach7815f562012-02-03 00:07:04 +00001900
Chris Lattnerc663a672010-10-01 05:51:02 +00001901 // No other transformations apply to volatile transfers.
Daniel Neilson8f30ec62018-05-11 14:30:02 +00001902 if (auto *M = dyn_cast<MemIntrinsic>(MI))
1903 if (M->isVolatile())
1904 return nullptr;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001905
1906 // If we have a memmove and the source operation is a constant global,
1907 // then the source and dest pointers can't alias, so we can change this
1908 // into a call to memcpy.
Daniel Neilson8f30ec62018-05-11 14:30:02 +00001909 if (auto *MMI = dyn_cast<AnyMemMoveInst>(MI)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001910 if (GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource()))
1911 if (GVSrc->isConstant()) {
Sanjay Patelaf674fb2015-12-14 17:24:23 +00001912 Module *M = CI.getModule();
Daniel Neilson8f30ec62018-05-11 14:30:02 +00001913 Intrinsic::ID MemCpyID =
1914 isa<AtomicMemMoveInst>(MMI)
1915 ? Intrinsic::memcpy_element_unordered_atomic
1916 : Intrinsic::memcpy;
Jay Foadb804a2b2011-07-12 14:06:48 +00001917 Type *Tys[3] = { CI.getArgOperand(0)->getType(),
1918 CI.getArgOperand(1)->getType(),
1919 CI.getArgOperand(2)->getType() };
Benjamin Kramere6e19332011-07-14 17:45:39 +00001920 CI.setCalledFunction(Intrinsic::getDeclaration(M, MemCpyID, Tys));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001921 Changed = true;
1922 }
1923 }
1924
Daniel Neilson8f30ec62018-05-11 14:30:02 +00001925 if (AnyMemTransferInst *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001926 // memmove(x,x,size) -> noop.
1927 if (MTI->getSource() == MTI->getDest())
Sanjay Patel4b198802016-02-01 22:23:39 +00001928 return eraseInstFromFunction(CI);
Eric Christopher7258dcd2010-04-16 23:37:20 +00001929 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001930
Eric Christopher7258dcd2010-04-16 23:37:20 +00001931 // If we can determine a pointer alignment that is bigger than currently
1932 // set, update the alignment.
Daniel Neilson8f30ec62018-05-11 14:30:02 +00001933 if (auto *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
1934 if (Instruction *I = SimplifyAnyMemTransfer(MTI))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001935 return I;
Daniel Neilsonf6651d42018-05-11 20:04:50 +00001936 } else if (auto *MSI = dyn_cast<AnyMemSetInst>(MI)) {
1937 if (Instruction *I = SimplifyAnyMemSet(MSI))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001938 return I;
1939 }
Gabor Greif590d95e2010-06-24 13:42:49 +00001940
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001941 if (Changed) return II;
1942 }
Jim Grosbach7815f562012-02-03 00:07:04 +00001943
Philip Reames68a2e4d2019-03-15 19:54:06 +00001944 // For vector result intrinsics, use the generic demanded vector support.
Philip Reamesc71e9962019-01-30 19:21:11 +00001945 if (II->getType()->isVectorTy()) {
1946 auto VWidth = II->getType()->getVectorNumElements();
1947 APInt UndefElts(VWidth, 0);
1948 APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth));
1949 if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) {
1950 if (V != II)
1951 return replaceInstUsesWith(*II, V);
1952 return II;
1953 }
1954 }
1955
Justin Lebar698c31b2017-01-27 00:58:58 +00001956 if (Instruction *I = SimplifyNVVMIntrinsic(II, *this))
1957 return I;
1958
Sanjay Patel1c600c62016-01-20 16:41:43 +00001959 auto SimplifyDemandedVectorEltsLow = [this](Value *Op, unsigned Width,
1960 unsigned DemandedWidth) {
Simon Pilgrim61116dd2015-09-17 20:32:45 +00001961 APInt UndefElts(Width, 0);
1962 APInt DemandedElts = APInt::getLowBitsSet(Width, DemandedWidth);
1963 return SimplifyDemandedVectorElts(Op, DemandedElts, UndefElts);
1964 };
1965
Sanjay Patel62f457b2019-05-06 15:35:02 +00001966 Intrinsic::ID IID = II->getIntrinsicID();
1967 switch (IID) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001968 default: break;
George Burgess IV3f089142016-12-20 23:46:36 +00001969 case Intrinsic::objectsize:
Erik Pilkington600e9de2019-01-30 20:34:35 +00001970 if (Value *V = lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/false))
1971 return replaceInstUsesWith(CI, V);
Craig Topperf40110f2014-04-25 05:29:35 +00001972 return nullptr;
Michael Ilseman536cc322012-12-13 03:13:36 +00001973 case Intrinsic::bswap: {
1974 Value *IIOperand = II->getArgOperand(0);
Craig Topperf40110f2014-04-25 05:29:35 +00001975 Value *X = nullptr;
Michael Ilseman536cc322012-12-13 03:13:36 +00001976
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001977 // bswap(trunc(bswap(x))) -> trunc(lshr(x, c))
Michael Ilseman536cc322012-12-13 03:13:36 +00001978 if (match(IIOperand, m_Trunc(m_BSwap(m_Value(X))))) {
1979 unsigned C = X->getType()->getPrimitiveSizeInBits() -
1980 IIOperand->getType()->getPrimitiveSizeInBits();
1981 Value *CV = ConstantInt::get(X->getType(), C);
Craig Topperbb4069e2017-07-07 23:16:26 +00001982 Value *V = Builder.CreateLShr(X, CV);
Michael Ilseman536cc322012-12-13 03:13:36 +00001983 return new TruncInst(V, IIOperand->getType());
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001984 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00001985 break;
Michael Ilseman536cc322012-12-13 03:13:36 +00001986 }
Sanjay Patelb695c552016-02-01 17:00:10 +00001987 case Intrinsic::masked_load:
Philip Reames7c8647b2019-04-25 01:18:56 +00001988 if (Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II))
Sanjay Patel4b198802016-02-01 22:23:39 +00001989 return replaceInstUsesWith(CI, SimplifiedMaskedOp);
Sanjay Patelb695c552016-02-01 17:00:10 +00001990 break;
Sanjay Patel04f792b2016-02-01 19:39:52 +00001991 case Intrinsic::masked_store:
Philip Reamese4588bb2019-03-20 18:44:58 +00001992 return simplifyMaskedStore(*II);
Sanjay Patel103ab7d2016-02-01 22:10:26 +00001993 case Intrinsic::masked_gather:
Philip Reames7c8647b2019-04-25 01:18:56 +00001994 return simplifyMaskedGather(*II);
Sanjay Patel103ab7d2016-02-01 22:10:26 +00001995 case Intrinsic::masked_scatter:
Philip Reamese4588bb2019-03-20 18:44:58 +00001996 return simplifyMaskedScatter(*II);
Piotr Padlewskic63b4922018-07-12 23:55:20 +00001997 case Intrinsic::launder_invariant_group:
1998 case Intrinsic::strip_invariant_group:
1999 if (auto *SkippedBarrier = simplifyInvariantGroupIntrinsic(*II, *this))
2000 return replaceInstUsesWith(*II, SkippedBarrier);
2001 break;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002002 case Intrinsic::powi:
Gabor Greif589a0b92010-06-24 12:58:35 +00002003 if (ConstantInt *Power = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
Philip Reames5000ba62017-12-27 01:14:30 +00002004 // 0 and 1 are handled in instsimplify
2005
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002006 // powi(x, -1) -> 1/x
Craig Topper79ab6432017-07-06 18:39:47 +00002007 if (Power->isMinusOne())
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002008 return BinaryOperator::CreateFDiv(ConstantFP::get(CI.getType(), 1.0),
Gabor Greif589a0b92010-06-24 12:58:35 +00002009 II->getArgOperand(0));
Philip Reamescd13a662017-12-27 01:30:12 +00002010 // powi(x, 2) -> x*x
2011 if (Power->equalsInt(2))
2012 return BinaryOperator::CreateFMul(II->getArgOperand(0),
2013 II->getArgOperand(0));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002014 }
2015 break;
Jim Grosbach7815f562012-02-03 00:07:04 +00002016
Sanjay Patel8e3ab172016-08-05 22:42:46 +00002017 case Intrinsic::cttz:
2018 case Intrinsic::ctlz:
Amaury Sechet763c59d2016-08-18 20:43:50 +00002019 if (auto *I = foldCttzCtlz(*II, *this))
2020 return I;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002021 break;
Sanjoy Dasb0984472015-04-08 04:27:22 +00002022
Craig Topper5b173f22017-06-21 16:32:35 +00002023 case Intrinsic::ctpop:
2024 if (auto *I = foldCtpop(*II, *this))
2025 return I;
2026 break;
2027
Sanjay Patela1395642018-11-13 23:27:23 +00002028 case Intrinsic::fshl:
2029 case Intrinsic::fshr: {
Sanjay Patelb3bcd952019-03-17 19:08:00 +00002030 Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);
2031 Type *Ty = II->getType();
2032 unsigned BitWidth = Ty->getScalarSizeInBits();
Sanjay Patelde1d5d32019-03-14 19:22:08 +00002033 Constant *ShAmtC;
2034 if (match(II->getArgOperand(2), m_Constant(ShAmtC)) &&
2035 !isa<ConstantExpr>(ShAmtC) && !ShAmtC->containsConstantExpression()) {
Sanjay Patelb3bcd952019-03-17 19:08:00 +00002036 // Canonicalize a shift amount constant operand to modulo the bit-width.
2037 Constant *WidthC = ConstantInt::get(Ty, BitWidth);
Sanjay Patelde1d5d32019-03-14 19:22:08 +00002038 Constant *ModuloC = ConstantExpr::getURem(ShAmtC, WidthC);
2039 if (ModuloC != ShAmtC) {
2040 II->setArgOperand(2, ModuloC);
2041 return II;
2042 }
Sanjay Patel60633932019-03-18 14:27:51 +00002043 assert(ConstantExpr::getICmp(ICmpInst::ICMP_UGT, WidthC, ShAmtC) ==
2044 ConstantInt::getTrue(CmpInst::makeCmpResultType(Ty)) &&
2045 "Shift amount expected to be modulo bitwidth");
2046
Sanjay Patel84de8a32019-03-18 14:10:11 +00002047 // Canonicalize funnel shift right by constant to funnel shift left. This
2048 // is not entirely arbitrary. For historical reasons, the backend may
2049 // recognize rotate left patterns but miss rotate right patterns.
Sanjay Patel62f457b2019-05-06 15:35:02 +00002050 if (IID == Intrinsic::fshr) {
Sanjay Patel84de8a32019-03-18 14:10:11 +00002051 // fshr X, Y, C --> fshl X, Y, (BitWidth - C)
Sanjay Patelb3bcd952019-03-17 19:08:00 +00002052 Constant *LeftShiftC = ConstantExpr::getSub(WidthC, ShAmtC);
2053 Module *Mod = II->getModule();
2054 Function *Fshl = Intrinsic::getDeclaration(Mod, Intrinsic::fshl, Ty);
Sanjay Patel84de8a32019-03-18 14:10:11 +00002055 return CallInst::Create(Fshl, { Op0, Op1, LeftShiftC });
Sanjay Patelb3bcd952019-03-17 19:08:00 +00002056 }
Sanjay Patel62f457b2019-05-06 15:35:02 +00002057 assert(IID == Intrinsic::fshl &&
Sanjay Patel84de8a32019-03-18 14:10:11 +00002058 "All funnel shifts by simple constants should go left");
Nikita Popov6e81d422018-11-23 22:45:08 +00002059
Sanjay Patel60633932019-03-18 14:27:51 +00002060 // fshl(X, 0, C) --> shl X, C
2061 // fshl(X, undef, C) --> shl X, C
2062 if (match(Op1, m_ZeroInt()) || match(Op1, m_Undef()))
2063 return BinaryOperator::CreateShl(Op0, ShAmtC);
Nikita Popov6e81d422018-11-23 22:45:08 +00002064
Sanjay Patel60633932019-03-18 14:27:51 +00002065 // fshl(0, X, C) --> lshr X, (BW-C)
2066 // fshl(undef, X, C) --> lshr X, (BW-C)
2067 if (match(Op0, m_ZeroInt()) || match(Op0, m_Undef()))
2068 return BinaryOperator::CreateLShr(Op1,
2069 ConstantExpr::getSub(WidthC, ShAmtC));
Nikita Popov6e81d422018-11-23 22:45:08 +00002070 }
2071
Nikita Popov5ecd6a42019-04-16 19:05:49 +00002072 // Left or right might be masked.
2073 if (SimplifyDemandedInstructionBits(*II))
2074 return &CI;
2075
Sanjay Patela1395642018-11-13 23:27:23 +00002076 // The shift amount (operand 2) of a funnel shift is modulo the bitwidth,
2077 // so only the low bits of the shift amount are demanded if the bitwidth is
2078 // a power-of-2.
Sanjay Patela1395642018-11-13 23:27:23 +00002079 if (!isPowerOf2_32(BitWidth))
2080 break;
2081 APInt Op2Demanded = APInt::getLowBitsSet(BitWidth, Log2_32_Ceil(BitWidth));
2082 KnownBits Op2Known(BitWidth);
2083 if (SimplifyDemandedBits(II, 2, Op2Demanded, Op2Known))
2084 return &CI;
2085 break;
2086 }
Nikita Popov37cf25c2019-03-20 18:00:27 +00002087 case Intrinsic::uadd_with_overflow:
Nikita Popov884feb12019-03-06 18:30:00 +00002088 case Intrinsic::sadd_with_overflow: {
2089 if (Instruction *I = canonicalizeConstantArg0ToArg1(CI))
2090 return I;
2091 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2092 return I;
2093
2094 // Given 2 constant operands whose sum does not overflow:
Nikita Popov37cf25c2019-03-20 18:00:27 +00002095 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
Nikita Popov884feb12019-03-06 18:30:00 +00002096 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
2097 Value *X;
2098 const APInt *C0, *C1;
2099 Value *Arg0 = II->getArgOperand(0);
2100 Value *Arg1 = II->getArgOperand(1);
Sanjay Patel62f457b2019-05-06 15:35:02 +00002101 bool IsSigned = IID == Intrinsic::sadd_with_overflow;
Nikita Popov37cf25c2019-03-20 18:00:27 +00002102 bool HasNWAdd = IsSigned ? match(Arg0, m_NSWAdd(m_Value(X), m_APInt(C0)))
2103 : match(Arg0, m_NUWAdd(m_Value(X), m_APInt(C0)));
2104 if (HasNWAdd && match(Arg1, m_APInt(C1))) {
Nikita Popov884feb12019-03-06 18:30:00 +00002105 bool Overflow;
Nikita Popov37cf25c2019-03-20 18:00:27 +00002106 APInt NewC =
2107 IsSigned ? C1->sadd_ov(*C0, Overflow) : C1->uadd_ov(*C0, Overflow);
Nikita Popov884feb12019-03-06 18:30:00 +00002108 if (!Overflow)
2109 return replaceInstUsesWith(
2110 *II, Builder.CreateBinaryIntrinsic(
Sanjay Patel62f457b2019-05-06 15:35:02 +00002111 IID, X, ConstantInt::get(Arg1->getType(), NewC)));
Nikita Popov884feb12019-03-06 18:30:00 +00002112 }
Nikita Popov884feb12019-03-06 18:30:00 +00002113 break;
2114 }
Nikita Popov7a543c32019-04-10 16:27:36 +00002115
Nick Lewyckyabe2cc12015-04-13 19:17:37 +00002116 case Intrinsic::umul_with_overflow:
2117 case Intrinsic::smul_with_overflow:
Sanjay Patel790af912018-11-26 22:00:41 +00002118 if (Instruction *I = canonicalizeConstantArg0ToArg1(CI))
2119 return I;
Justin Bognercd1d5aa2016-08-17 20:30:52 +00002120 LLVM_FALLTHROUGH;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002121
Nick Lewyckyabe2cc12015-04-13 19:17:37 +00002122 case Intrinsic::usub_with_overflow:
Nikita Popov7a543c32019-04-10 16:27:36 +00002123 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2124 return I;
2125 break;
2126
Nick Lewyckyabe2cc12015-04-13 19:17:37 +00002127 case Intrinsic::ssub_with_overflow: {
Nikita Popov884feb12019-03-06 18:30:00 +00002128 if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
2129 return I;
Benjamin Kramera420df22014-07-04 10:22:21 +00002130
Nikita Popov7a543c32019-04-10 16:27:36 +00002131 Constant *C;
2132 Value *Arg0 = II->getArgOperand(0);
2133 Value *Arg1 = II->getArgOperand(1);
2134 // Given a constant C that is not the minimum signed value
2135 // for an integer of a given bit width:
2136 //
2137 // ssubo X, C -> saddo X, -C
2138 if (match(Arg1, m_Constant(C)) && C->isNotMinSignedValue()) {
2139 Value *NegVal = ConstantExpr::getNeg(C);
2140 // Build a saddo call that is equivalent to the discovered
2141 // ssubo call.
2142 return replaceInstUsesWith(
2143 *II, Builder.CreateBinaryIntrinsic(Intrinsic::sadd_with_overflow,
2144 Arg0, NegVal));
2145 }
2146
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002147 break;
Erik Eckstein096ff7d2014-12-11 08:02:30 +00002148 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002149
Nikita Popov085d24a2018-11-28 16:36:52 +00002150 case Intrinsic::uadd_sat:
2151 case Intrinsic::sadd_sat:
2152 if (Instruction *I = canonicalizeConstantArg0ToArg1(CI))
2153 return I;
Nikita Popov78a92952018-11-28 16:36:59 +00002154 LLVM_FALLTHROUGH;
2155 case Intrinsic::usub_sat:
2156 case Intrinsic::ssub_sat: {
2157 Value *Arg0 = II->getArgOperand(0);
2158 Value *Arg1 = II->getArgOperand(1);
Nikita Popov78a92952018-11-28 16:36:59 +00002159
2160 // Make use of known overflow information.
2161 OverflowResult OR;
2162 switch (IID) {
2163 default:
2164 llvm_unreachable("Unexpected intrinsic!");
2165 case Intrinsic::uadd_sat:
2166 OR = computeOverflowForUnsignedAdd(Arg0, Arg1, II);
2167 if (OR == OverflowResult::NeverOverflows)
2168 return BinaryOperator::CreateNUWAdd(Arg0, Arg1);
2169 if (OR == OverflowResult::AlwaysOverflows)
2170 return replaceInstUsesWith(*II,
2171 ConstantInt::getAllOnesValue(II->getType()));
2172 break;
2173 case Intrinsic::usub_sat:
2174 OR = computeOverflowForUnsignedSub(Arg0, Arg1, II);
2175 if (OR == OverflowResult::NeverOverflows)
2176 return BinaryOperator::CreateNUWSub(Arg0, Arg1);
2177 if (OR == OverflowResult::AlwaysOverflows)
2178 return replaceInstUsesWith(*II,
2179 ConstantInt::getNullValue(II->getType()));
2180 break;
2181 case Intrinsic::sadd_sat:
2182 if (willNotOverflowSignedAdd(Arg0, Arg1, *II))
2183 return BinaryOperator::CreateNSWAdd(Arg0, Arg1);
2184 break;
2185 case Intrinsic::ssub_sat:
2186 if (willNotOverflowSignedSub(Arg0, Arg1, *II))
2187 return BinaryOperator::CreateNSWSub(Arg0, Arg1);
2188 break;
2189 }
Nikita Popov42f89982018-11-28 16:37:09 +00002190
2191 // ssub.sat(X, C) -> sadd.sat(X, -C) if C != MIN
Nikita Popov0c5d6cc2018-12-01 10:58:34 +00002192 Constant *C;
2193 if (IID == Intrinsic::ssub_sat && match(Arg1, m_Constant(C)) &&
2194 C->isNotMinSignedValue()) {
2195 Value *NegVal = ConstantExpr::getNeg(C);
Nikita Popov42f89982018-11-28 16:37:09 +00002196 return replaceInstUsesWith(
2197 *II, Builder.CreateBinaryIntrinsic(
2198 Intrinsic::sadd_sat, Arg0, NegVal));
2199 }
Nikita Popov8d63aed2018-11-28 16:37:15 +00002200
2201 // sat(sat(X + Val2) + Val) -> sat(X + (Val+Val2))
2202 // sat(sat(X - Val2) - Val) -> sat(X - (Val+Val2))
2203 // if Val and Val2 have the same sign
2204 if (auto *Other = dyn_cast<IntrinsicInst>(Arg0)) {
2205 Value *X;
2206 const APInt *Val, *Val2;
2207 APInt NewVal;
2208 bool IsUnsigned =
2209 IID == Intrinsic::uadd_sat || IID == Intrinsic::usub_sat;
Sanjay Patel62f457b2019-05-06 15:35:02 +00002210 if (Other->getIntrinsicID() == IID &&
Nikita Popov8d63aed2018-11-28 16:37:15 +00002211 match(Arg1, m_APInt(Val)) &&
2212 match(Other->getArgOperand(0), m_Value(X)) &&
2213 match(Other->getArgOperand(1), m_APInt(Val2))) {
2214 if (IsUnsigned)
2215 NewVal = Val->uadd_sat(*Val2);
2216 else if (Val->isNonNegative() == Val2->isNonNegative()) {
2217 bool Overflow;
2218 NewVal = Val->sadd_ov(*Val2, Overflow);
2219 if (Overflow) {
2220 // Both adds together may add more than SignedMaxValue
2221 // without saturating the final result.
2222 break;
2223 }
2224 } else {
2225 // Cannot fold saturated addition with different signs.
2226 break;
2227 }
2228
2229 return replaceInstUsesWith(
2230 *II, Builder.CreateBinaryIntrinsic(
2231 IID, X, ConstantInt::get(II->getType(), NewVal)));
2232 }
2233 }
Nikita Popov085d24a2018-11-28 16:36:52 +00002234 break;
Nikita Popov78a92952018-11-28 16:36:59 +00002235 }
Nikita Popov085d24a2018-11-28 16:36:52 +00002236
Matt Arsenaultd6511b42014-10-21 23:00:20 +00002237 case Intrinsic::minnum:
Thomas Livelyc3392502018-10-19 19:01:26 +00002238 case Intrinsic::maxnum:
2239 case Intrinsic::minimum:
2240 case Intrinsic::maximum: {
Sanjay Patel790af912018-11-26 22:00:41 +00002241 if (Instruction *I = canonicalizeConstantArg0ToArg1(CI))
2242 return I;
Matt Arsenaultd6511b42014-10-21 23:00:20 +00002243 Value *Arg0 = II->getArgOperand(0);
2244 Value *Arg1 = II->getArgOperand(1);
Sanjay Patelc7bb1432018-05-10 20:03:13 +00002245 Value *X, *Y;
2246 if (match(Arg0, m_FNeg(m_Value(X))) && match(Arg1, m_FNeg(m_Value(Y))) &&
2247 (Arg0->hasOneUse() || Arg1->hasOneUse())) {
2248 // If both operands are negated, invert the call and negate the result:
Thomas Livelyc3392502018-10-19 19:01:26 +00002249 // min(-X, -Y) --> -(max(X, Y))
2250 // max(-X, -Y) --> -(min(X, Y))
2251 Intrinsic::ID NewIID;
Volkan Keles3ca146d2018-10-31 17:50:52 +00002252 switch (IID) {
Thomas Livelyc3392502018-10-19 19:01:26 +00002253 case Intrinsic::maxnum:
2254 NewIID = Intrinsic::minnum;
2255 break;
2256 case Intrinsic::minnum:
2257 NewIID = Intrinsic::maxnum;
2258 break;
2259 case Intrinsic::maximum:
2260 NewIID = Intrinsic::minimum;
2261 break;
2262 case Intrinsic::minimum:
2263 NewIID = Intrinsic::maximum;
2264 break;
2265 default:
2266 llvm_unreachable("unexpected intrinsic ID");
2267 }
Neil Henning57f5d0a2018-10-08 10:32:33 +00002268 Value *NewCall = Builder.CreateBinaryIntrinsic(NewIID, X, Y, II);
Sanjay Patelc7bb1432018-05-10 20:03:13 +00002269 Instruction *FNeg = BinaryOperator::CreateFNeg(NewCall);
2270 FNeg->copyIRFlags(II);
2271 return FNeg;
2272 }
Volkan Keles3ca146d2018-10-31 17:50:52 +00002273
2274 // m(m(X, C2), C1) -> m(X, C)
2275 const APFloat *C1, *C2;
2276 if (auto *M = dyn_cast<IntrinsicInst>(Arg0)) {
2277 if (M->getIntrinsicID() == IID && match(Arg1, m_APFloat(C1)) &&
2278 ((match(M->getArgOperand(0), m_Value(X)) &&
2279 match(M->getArgOperand(1), m_APFloat(C2))) ||
2280 (match(M->getArgOperand(1), m_Value(X)) &&
2281 match(M->getArgOperand(0), m_APFloat(C2))))) {
2282 APFloat Res(0.0);
2283 switch (IID) {
2284 case Intrinsic::maxnum:
2285 Res = maxnum(*C1, *C2);
2286 break;
2287 case Intrinsic::minnum:
2288 Res = minnum(*C1, *C2);
2289 break;
2290 case Intrinsic::maximum:
2291 Res = maximum(*C1, *C2);
2292 break;
2293 case Intrinsic::minimum:
2294 Res = minimum(*C1, *C2);
2295 break;
2296 default:
2297 llvm_unreachable("unexpected intrinsic ID");
2298 }
2299 Instruction *NewCall = Builder.CreateBinaryIntrinsic(
2300 IID, X, ConstantFP::get(Arg0->getType(), Res));
2301 NewCall->copyIRFlags(II);
2302 return replaceInstUsesWith(*II, NewCall);
2303 }
2304 }
2305
Matt Arsenaultd6511b42014-10-21 23:00:20 +00002306 break;
2307 }
Matt Arsenault1cc294c2017-01-03 04:32:31 +00002308 case Intrinsic::fmuladd: {
Matt Arsenault92057602017-02-16 18:46:24 +00002309 // Canonicalize fast fmuladd to the separate fmul + fadd.
Sanjay Patel629c4112017-11-06 16:27:15 +00002310 if (II->isFast()) {
Craig Topperbb4069e2017-07-07 23:16:26 +00002311 BuilderTy::FastMathFlagGuard Guard(Builder);
2312 Builder.setFastMathFlags(II->getFastMathFlags());
2313 Value *Mul = Builder.CreateFMul(II->getArgOperand(0),
2314 II->getArgOperand(1));
2315 Value *Add = Builder.CreateFAdd(Mul, II->getArgOperand(2));
Matt Arsenault92057602017-02-16 18:46:24 +00002316 Add->takeName(II);
2317 return replaceInstUsesWith(*II, Add);
2318 }
2319
2320 LLVM_FALLTHROUGH;
2321 }
2322 case Intrinsic::fma: {
Sanjay Patel790af912018-11-26 22:00:41 +00002323 if (Instruction *I = canonicalizeConstantArg0ToArg1(CI))
2324 return I;
Matt Arsenaultb264c942017-01-03 04:32:35 +00002325
Matt Arsenault1cc294c2017-01-03 04:32:31 +00002326 // fma fneg(x), fneg(y), z -> fma x, y, z
Sanjay Patel790af912018-11-26 22:00:41 +00002327 Value *Src0 = II->getArgOperand(0);
2328 Value *Src1 = II->getArgOperand(1);
Sanjay Patel236442e2018-04-05 13:24:26 +00002329 Value *X, *Y;
2330 if (match(Src0, m_FNeg(m_Value(X))) && match(Src1, m_FNeg(m_Value(Y)))) {
2331 II->setArgOperand(0, X);
2332 II->setArgOperand(1, Y);
Matt Arsenault3f509042017-01-10 23:17:52 +00002333 return II;
Matt Arsenault1cc294c2017-01-03 04:32:31 +00002334 }
2335
2336 // fma fabs(x), fabs(x), z -> fma x, x, z
Matt Arsenaultd1496502018-07-27 09:04:35 +00002337 if (match(Src0, m_FAbs(m_Value(X))) &&
2338 match(Src1, m_FAbs(m_Specific(X)))) {
Sanjay Patel236442e2018-04-05 13:24:26 +00002339 II->setArgOperand(0, X);
2340 II->setArgOperand(1, X);
Matt Arsenault3f509042017-01-10 23:17:52 +00002341 return II;
Matt Arsenault1cc294c2017-01-03 04:32:31 +00002342 }
2343
Matt Arsenaultb264c942017-01-03 04:32:35 +00002344 // fma x, 1, z -> fadd x, z
2345 if (match(Src1, m_FPOne())) {
Sanjay Patel236442e2018-04-05 13:24:26 +00002346 auto *FAdd = BinaryOperator::CreateFAdd(Src0, II->getArgOperand(2));
2347 FAdd->copyFastMathFlags(II);
2348 return FAdd;
Matt Arsenaultb264c942017-01-03 04:32:35 +00002349 }
2350
Matt Arsenault1cc294c2017-01-03 04:32:31 +00002351 break;
2352 }
Matt Arsenault56ff4832017-01-03 22:40:34 +00002353 case Intrinsic::fabs: {
2354 Value *Cond;
2355 Constant *LHS, *RHS;
2356 if (match(II->getArgOperand(0),
2357 m_Select(m_Value(Cond), m_Constant(LHS), m_Constant(RHS)))) {
Craig Topperbb4069e2017-07-07 23:16:26 +00002358 CallInst *Call0 = Builder.CreateCall(II->getCalledFunction(), {LHS});
2359 CallInst *Call1 = Builder.CreateCall(II->getCalledFunction(), {RHS});
Matt Arsenault56ff4832017-01-03 22:40:34 +00002360 return SelectInst::Create(Cond, Call0, Call1);
2361 }
2362
Matt Arsenault954a6242017-01-23 23:55:08 +00002363 LLVM_FALLTHROUGH;
2364 }
2365 case Intrinsic::ceil:
2366 case Intrinsic::floor:
2367 case Intrinsic::round:
2368 case Intrinsic::nearbyint:
Joerg Sonnenberger28bed102017-03-31 19:58:07 +00002369 case Intrinsic::rint:
Matt Arsenault954a6242017-01-23 23:55:08 +00002370 case Intrinsic::trunc: {
Matt Arsenault72333442017-01-17 00:10:40 +00002371 Value *ExtSrc;
Sanjay Patel32381d72018-03-23 21:18:12 +00002372 if (match(II->getArgOperand(0), m_OneUse(m_FPExt(m_Value(ExtSrc))))) {
2373 // Narrow the call: intrinsic (fpext x) -> fpext (intrinsic x)
Sanjay Patel62f457b2019-05-06 15:35:02 +00002374 Value *NarrowII = Builder.CreateUnaryIntrinsic(IID, ExtSrc, II);
Sanjay Patel32381d72018-03-23 21:18:12 +00002375 return new FPExtInst(NarrowII, II->getType());
Matt Arsenault72333442017-01-17 00:10:40 +00002376 }
Matt Arsenault56ff4832017-01-03 22:40:34 +00002377 break;
2378 }
Matt Arsenault3bdd75d2017-01-04 22:49:03 +00002379 case Intrinsic::cos:
2380 case Intrinsic::amdgcn_cos: {
Sanjay Patel0f29e952018-08-29 18:27:49 +00002381 Value *X;
Matt Arsenault3bdd75d2017-01-04 22:49:03 +00002382 Value *Src = II->getArgOperand(0);
Sanjay Patel0f29e952018-08-29 18:27:49 +00002383 if (match(Src, m_FNeg(m_Value(X))) || match(Src, m_FAbs(m_Value(X)))) {
Matt Arsenault3bdd75d2017-01-04 22:49:03 +00002384 // cos(-x) -> cos(x)
2385 // cos(fabs(x)) -> cos(x)
Sanjay Patel0f29e952018-08-29 18:27:49 +00002386 II->setArgOperand(0, X);
Matt Arsenault3bdd75d2017-01-04 22:49:03 +00002387 return II;
2388 }
Sanjay Patel0f29e952018-08-29 18:27:49 +00002389 break;
2390 }
2391 case Intrinsic::sin: {
2392 Value *X;
2393 if (match(II->getArgOperand(0), m_OneUse(m_FNeg(m_Value(X))))) {
2394 // sin(-x) --> -sin(x)
Neil Henning57f5d0a2018-10-08 10:32:33 +00002395 Value *NewSin = Builder.CreateUnaryIntrinsic(Intrinsic::sin, X, II);
Sanjay Patel0f29e952018-08-29 18:27:49 +00002396 Instruction *FNeg = BinaryOperator::CreateFNeg(NewSin);
2397 FNeg->copyFastMathFlags(II);
2398 return FNeg;
2399 }
Matt Arsenault3bdd75d2017-01-04 22:49:03 +00002400 break;
2401 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002402 case Intrinsic::ppc_altivec_lvx:
2403 case Intrinsic::ppc_altivec_lvxl:
Bill Wendlingb902f1d2011-04-13 00:36:11 +00002404 // Turn PPC lvx -> load if the pointer is known aligned.
Daniel Jasperaec2fa32016-12-19 08:22:17 +00002405 if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, &AC,
Justin Bogner99798402016-08-05 01:06:44 +00002406 &DT) >= 16) {
Craig Topperbb4069e2017-07-07 23:16:26 +00002407 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002408 PointerType::getUnqual(II->getType()));
James Y Knight14359ef2019-02-01 20:44:24 +00002409 return new LoadInst(II->getType(), Ptr);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002410 }
2411 break;
Bill Schmidt72954782014-11-12 04:19:40 +00002412 case Intrinsic::ppc_vsx_lxvw4x:
2413 case Intrinsic::ppc_vsx_lxvd2x: {
2414 // Turn PPC VSX loads into normal loads.
Craig Topperbb4069e2017-07-07 23:16:26 +00002415 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
2416 PointerType::getUnqual(II->getType()));
James Y Knight14359ef2019-02-01 20:44:24 +00002417 return new LoadInst(II->getType(), Ptr, Twine(""), false, 1);
Bill Schmidt72954782014-11-12 04:19:40 +00002418 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002419 case Intrinsic::ppc_altivec_stvx:
2420 case Intrinsic::ppc_altivec_stvxl:
2421 // Turn stvx -> store if the pointer is known aligned.
Daniel Jasperaec2fa32016-12-19 08:22:17 +00002422 if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, &AC,
Justin Bogner99798402016-08-05 01:06:44 +00002423 &DT) >= 16) {
Jim Grosbach7815f562012-02-03 00:07:04 +00002424 Type *OpPtrTy =
Gabor Greifa6d75e22010-06-24 15:51:11 +00002425 PointerType::getUnqual(II->getArgOperand(0)->getType());
Craig Topperbb4069e2017-07-07 23:16:26 +00002426 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
Gabor Greifa6d75e22010-06-24 15:51:11 +00002427 return new StoreInst(II->getArgOperand(0), Ptr);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002428 }
2429 break;
Bill Schmidt72954782014-11-12 04:19:40 +00002430 case Intrinsic::ppc_vsx_stxvw4x:
2431 case Intrinsic::ppc_vsx_stxvd2x: {
2432 // Turn PPC VSX stores into normal stores.
2433 Type *OpPtrTy = PointerType::getUnqual(II->getArgOperand(0)->getType());
Craig Topperbb4069e2017-07-07 23:16:26 +00002434 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
Bill Schmidt72954782014-11-12 04:19:40 +00002435 return new StoreInst(II->getArgOperand(0), Ptr, false, 1);
2436 }
Hal Finkel221f4672015-02-26 18:56:03 +00002437 case Intrinsic::ppc_qpx_qvlfs:
2438 // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
Daniel Jasperaec2fa32016-12-19 08:22:17 +00002439 if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, &AC,
Justin Bogner99798402016-08-05 01:06:44 +00002440 &DT) >= 16) {
Craig Topperbb4069e2017-07-07 23:16:26 +00002441 Type *VTy = VectorType::get(Builder.getFloatTy(),
Hal Finkelf0d68d72015-05-11 06:37:03 +00002442 II->getType()->getVectorNumElements());
Craig Topperbb4069e2017-07-07 23:16:26 +00002443 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
Hal Finkelf0d68d72015-05-11 06:37:03 +00002444 PointerType::getUnqual(VTy));
James Y Knight14359ef2019-02-01 20:44:24 +00002445 Value *Load = Builder.CreateLoad(VTy, Ptr);
Hal Finkelf0d68d72015-05-11 06:37:03 +00002446 return new FPExtInst(Load, II->getType());
Hal Finkel221f4672015-02-26 18:56:03 +00002447 }
2448 break;
2449 case Intrinsic::ppc_qpx_qvlfd:
2450 // Turn PPC QPX qvlfd -> load if the pointer is known aligned.
Daniel Jasperaec2fa32016-12-19 08:22:17 +00002451 if (getOrEnforceKnownAlignment(II->getArgOperand(0), 32, DL, II, &AC,
Justin Bogner99798402016-08-05 01:06:44 +00002452 &DT) >= 32) {
Craig Topperbb4069e2017-07-07 23:16:26 +00002453 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
Hal Finkel221f4672015-02-26 18:56:03 +00002454 PointerType::getUnqual(II->getType()));
James Y Knight14359ef2019-02-01 20:44:24 +00002455 return new LoadInst(II->getType(), Ptr);
Hal Finkel221f4672015-02-26 18:56:03 +00002456 }
2457 break;
2458 case Intrinsic::ppc_qpx_qvstfs:
2459 // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
Daniel Jasperaec2fa32016-12-19 08:22:17 +00002460 if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, &AC,
Justin Bogner99798402016-08-05 01:06:44 +00002461 &DT) >= 16) {
Craig Topperbb4069e2017-07-07 23:16:26 +00002462 Type *VTy = VectorType::get(Builder.getFloatTy(),
Hal Finkelf0d68d72015-05-11 06:37:03 +00002463 II->getArgOperand(0)->getType()->getVectorNumElements());
Craig Topperbb4069e2017-07-07 23:16:26 +00002464 Value *TOp = Builder.CreateFPTrunc(II->getArgOperand(0), VTy);
Hal Finkelf0d68d72015-05-11 06:37:03 +00002465 Type *OpPtrTy = PointerType::getUnqual(VTy);
Craig Topperbb4069e2017-07-07 23:16:26 +00002466 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
Hal Finkelf0d68d72015-05-11 06:37:03 +00002467 return new StoreInst(TOp, Ptr);
Hal Finkel221f4672015-02-26 18:56:03 +00002468 }
2469 break;
2470 case Intrinsic::ppc_qpx_qvstfd:
2471 // Turn PPC QPX qvstfd -> store if the pointer is known aligned.
Daniel Jasperaec2fa32016-12-19 08:22:17 +00002472 if (getOrEnforceKnownAlignment(II->getArgOperand(1), 32, DL, II, &AC,
Justin Bogner99798402016-08-05 01:06:44 +00002473 &DT) >= 32) {
Hal Finkel221f4672015-02-26 18:56:03 +00002474 Type *OpPtrTy =
2475 PointerType::getUnqual(II->getArgOperand(0)->getType());
Craig Topperbb4069e2017-07-07 23:16:26 +00002476 Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
Hal Finkel221f4672015-02-26 18:56:03 +00002477 return new StoreInst(II->getArgOperand(0), Ptr);
2478 }
2479 break;
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002480
Craig Topper83240032017-07-31 18:52:13 +00002481 case Intrinsic::x86_bmi_bextr_32:
2482 case Intrinsic::x86_bmi_bextr_64:
2483 case Intrinsic::x86_tbm_bextri_u32:
2484 case Intrinsic::x86_tbm_bextri_u64:
2485 // If the RHS is a constant we can try some simplifications.
2486 if (auto *C = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
2487 uint64_t Shift = C->getZExtValue();
2488 uint64_t Length = (Shift >> 8) & 0xff;
2489 Shift &= 0xff;
2490 unsigned BitWidth = II->getType()->getIntegerBitWidth();
2491 // If the length is 0 or the shift is out of range, replace with zero.
2492 if (Length == 0 || Shift >= BitWidth)
2493 return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), 0));
2494 // If the LHS is also a constant, we can completely constant fold this.
2495 if (auto *InC = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
2496 uint64_t Result = InC->getZExtValue() >> Shift;
2497 if (Length > BitWidth)
2498 Length = BitWidth;
2499 Result &= maskTrailingOnes<uint64_t>(Length);
2500 return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Result));
2501 }
2502 // TODO should we turn this into 'and' if shift is 0? Or 'shl' if we
2503 // are only masking bits that a shift already cleared?
2504 }
2505 break;
2506
Craig Topper317a51e2017-07-31 18:52:15 +00002507 case Intrinsic::x86_bmi_bzhi_32:
2508 case Intrinsic::x86_bmi_bzhi_64:
2509 // If the RHS is a constant we can try some simplifications.
2510 if (auto *C = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
2511 uint64_t Index = C->getZExtValue() & 0xff;
2512 unsigned BitWidth = II->getType()->getIntegerBitWidth();
2513 if (Index >= BitWidth)
2514 return replaceInstUsesWith(CI, II->getArgOperand(0));
2515 if (Index == 0)
2516 return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), 0));
2517 // If the LHS is also a constant, we can completely constant fold this.
2518 if (auto *InC = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
2519 uint64_t Result = InC->getZExtValue();
2520 Result &= maskTrailingOnes<uint64_t>(Index);
2521 return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Result));
2522 }
2523 // TODO should we convert this to an AND if the RHS is constant?
2524 }
2525 break;
2526
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002527 case Intrinsic::x86_vcvtph2ps_128:
2528 case Intrinsic::x86_vcvtph2ps_256: {
2529 auto Arg = II->getArgOperand(0);
2530 auto ArgType = cast<VectorType>(Arg->getType());
2531 auto RetType = cast<VectorType>(II->getType());
2532 unsigned ArgWidth = ArgType->getNumElements();
2533 unsigned RetWidth = RetType->getNumElements();
2534 assert(RetWidth <= ArgWidth && "Unexpected input/return vector widths");
2535 assert(ArgType->isIntOrIntVectorTy() &&
2536 ArgType->getScalarSizeInBits() == 16 &&
2537 "CVTPH2PS input type should be 16-bit integer vector");
2538 assert(RetType->getScalarType()->isFloatTy() &&
2539 "CVTPH2PS output type should be 32-bit float vector");
2540
2541 // Constant folding: Convert to generic half to single conversion.
Simon Pilgrim48ffca02015-09-12 14:00:17 +00002542 if (isa<ConstantAggregateZero>(Arg))
Sanjay Patel4b198802016-02-01 22:23:39 +00002543 return replaceInstUsesWith(*II, ConstantAggregateZero::get(RetType));
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002544
Simon Pilgrim48ffca02015-09-12 14:00:17 +00002545 if (isa<ConstantDataVector>(Arg)) {
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002546 auto VectorHalfAsShorts = Arg;
2547 if (RetWidth < ArgWidth) {
Craig Topper99d1eab2016-06-12 00:41:19 +00002548 SmallVector<uint32_t, 8> SubVecMask;
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002549 for (unsigned i = 0; i != RetWidth; ++i)
2550 SubVecMask.push_back((int)i);
Craig Topperbb4069e2017-07-07 23:16:26 +00002551 VectorHalfAsShorts = Builder.CreateShuffleVector(
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002552 Arg, UndefValue::get(ArgType), SubVecMask);
2553 }
2554
2555 auto VectorHalfType =
2556 VectorType::get(Type::getHalfTy(II->getContext()), RetWidth);
2557 auto VectorHalfs =
Craig Topperbb4069e2017-07-07 23:16:26 +00002558 Builder.CreateBitCast(VectorHalfAsShorts, VectorHalfType);
2559 auto VectorFloats = Builder.CreateFPExt(VectorHalfs, RetType);
Sanjay Patel4b198802016-02-01 22:23:39 +00002560 return replaceInstUsesWith(*II, VectorFloats);
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002561 }
2562
2563 // We only use the lowest lanes of the argument.
Simon Pilgrim996725e2015-09-19 11:41:53 +00002564 if (Value *V = SimplifyDemandedVectorEltsLow(Arg, ArgWidth, RetWidth)) {
Simon Pilgrim20c607b2015-09-12 13:39:53 +00002565 II->setArgOperand(0, V);
2566 return II;
2567 }
2568 break;
2569 }
2570
Chandler Carruthcf414cf2011-01-10 07:19:37 +00002571 case Intrinsic::x86_sse_cvtss2si:
2572 case Intrinsic::x86_sse_cvtss2si64:
2573 case Intrinsic::x86_sse_cvttss2si:
2574 case Intrinsic::x86_sse_cvttss2si64:
2575 case Intrinsic::x86_sse2_cvtsd2si:
2576 case Intrinsic::x86_sse2_cvtsd2si64:
2577 case Intrinsic::x86_sse2_cvttsd2si:
Craig Topperaeaa52c2016-12-14 07:46:12 +00002578 case Intrinsic::x86_sse2_cvttsd2si64:
2579 case Intrinsic::x86_avx512_vcvtss2si32:
2580 case Intrinsic::x86_avx512_vcvtss2si64:
2581 case Intrinsic::x86_avx512_vcvtss2usi32:
2582 case Intrinsic::x86_avx512_vcvtss2usi64:
2583 case Intrinsic::x86_avx512_vcvtsd2si32:
2584 case Intrinsic::x86_avx512_vcvtsd2si64:
2585 case Intrinsic::x86_avx512_vcvtsd2usi32:
2586 case Intrinsic::x86_avx512_vcvtsd2usi64:
2587 case Intrinsic::x86_avx512_cvttss2si:
2588 case Intrinsic::x86_avx512_cvttss2si64:
2589 case Intrinsic::x86_avx512_cvttss2usi:
2590 case Intrinsic::x86_avx512_cvttss2usi64:
2591 case Intrinsic::x86_avx512_cvttsd2si:
2592 case Intrinsic::x86_avx512_cvttsd2si64:
2593 case Intrinsic::x86_avx512_cvttsd2usi:
2594 case Intrinsic::x86_avx512_cvttsd2usi64: {
Chandler Carruthcf414cf2011-01-10 07:19:37 +00002595 // These intrinsics only demand the 0th element of their input vectors. If
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002596 // we can simplify the input based on that, do so now.
Simon Pilgrim996725e2015-09-19 11:41:53 +00002597 Value *Arg = II->getArgOperand(0);
2598 unsigned VWidth = Arg->getType()->getVectorNumElements();
2599 if (Value *V = SimplifyDemandedVectorEltsLow(Arg, VWidth, 1)) {
Gabor Greif5b1370e2010-06-28 16:50:57 +00002600 II->setArgOperand(0, V);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00002601 return II;
2602 }
Simon Pilgrim18617d12015-08-05 08:18:00 +00002603 break;
2604 }
2605
Mikhail Dvoretckii8393f902018-06-19 10:49:12 +00002606 case Intrinsic::x86_sse41_round_ps:
2607 case Intrinsic::x86_sse41_round_pd:
2608 case Intrinsic::x86_avx_round_ps_256:
2609 case Intrinsic::x86_avx_round_pd_256:
2610 case Intrinsic::x86_avx512_mask_rndscale_ps_128:
2611 case Intrinsic::x86_avx512_mask_rndscale_ps_256:
2612 case Intrinsic::x86_avx512_mask_rndscale_ps_512:
2613 case Intrinsic::x86_avx512_mask_rndscale_pd_128:
2614 case Intrinsic::x86_avx512_mask_rndscale_pd_256:
2615 case Intrinsic::x86_avx512_mask_rndscale_pd_512:
2616 case Intrinsic::x86_avx512_mask_rndscale_ss:
2617 case Intrinsic::x86_avx512_mask_rndscale_sd:
2618 if (Value *V = simplifyX86round(*II, Builder))
2619 return replaceInstUsesWith(*II, V);
2620 break;
2621
Simon Pilgrim91e3ac82016-06-07 08:18:35 +00002622 case Intrinsic::x86_mmx_pmovmskb:
2623 case Intrinsic::x86_sse_movmsk_ps:
2624 case Intrinsic::x86_sse2_movmsk_pd:
2625 case Intrinsic::x86_sse2_pmovmskb_128:
2626 case Intrinsic::x86_avx_movmsk_pd_256:
2627 case Intrinsic::x86_avx_movmsk_ps_256:
Eugene Zelenko7f0f9bc2017-10-24 21:24:53 +00002628 case Intrinsic::x86_avx2_pmovmskb:
Sanjay Patel2aa2dc72018-12-11 16:38:03 +00002629 if (Value *V = simplifyX86movmsk(*II, Builder))
Simon Pilgrim91e3ac82016-06-07 08:18:35 +00002630 return replaceInstUsesWith(*II, V);
2631 break;
Simon Pilgrim91e3ac82016-06-07 08:18:35 +00002632
Simon Pilgrim471efd22016-02-20 23:17:35 +00002633 case Intrinsic::x86_sse_comieq_ss:
2634 case Intrinsic::x86_sse_comige_ss:
2635 case Intrinsic::x86_sse_comigt_ss:
2636 case Intrinsic::x86_sse_comile_ss:
2637 case Intrinsic::x86_sse_comilt_ss:
2638 case Intrinsic::x86_sse_comineq_ss:
2639 case Intrinsic::x86_sse_ucomieq_ss:
2640 case Intrinsic::x86_sse_ucomige_ss:
2641 case Intrinsic::x86_sse_ucomigt_ss:
2642 case Intrinsic::x86_sse_ucomile_ss:
2643 case Intrinsic::x86_sse_ucomilt_ss:
2644 case Intrinsic::x86_sse_ucomineq_ss:
2645 case Intrinsic::x86_sse2_comieq_sd:
2646 case Intrinsic::x86_sse2_comige_sd:
2647 case Intrinsic::x86_sse2_comigt_sd:
2648 case Intrinsic::x86_sse2_comile_sd:
2649 case Intrinsic::x86_sse2_comilt_sd:
2650 case Intrinsic::x86_sse2_comineq_sd:
2651 case Intrinsic::x86_sse2_ucomieq_sd:
2652 case Intrinsic::x86_sse2_ucomige_sd:
2653 case Intrinsic::x86_sse2_ucomigt_sd:
2654 case Intrinsic::x86_sse2_ucomile_sd:
2655 case Intrinsic::x86_sse2_ucomilt_sd:
Craig Topperd9639532016-12-11 07:42:04 +00002656 case Intrinsic::x86_sse2_ucomineq_sd:
Craig Topperd00db692016-12-31 00:45:06 +00002657 case Intrinsic::x86_avx512_vcomi_ss:
2658 case Intrinsic::x86_avx512_vcomi_sd:
Craig Topperd9639532016-12-11 07:42:04 +00002659 case Intrinsic::x86_avx512_mask_cmp_ss:
2660 case Intrinsic::x86_avx512_mask_cmp_sd: {
Simon Pilgrim471efd22016-02-20 23:17:35 +00002661 // These intrinsics only demand the 0th element of their input vectors. If
2662 // we can simplify the input based on that, do so now.
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00002663 bool MadeChange = false;
Simon Pilgrim471efd22016-02-20 23:17:35 +00002664 Value *Arg0 = II->getArgOperand(0);
2665 Value *Arg1 = II->getArgOperand(1);
2666 unsigned VWidth = Arg0->getType()->getVectorNumElements();
2667 if (Value *V = SimplifyDemandedVectorEltsLow(Arg0, VWidth, 1)) {
2668 II->setArgOperand(0, V);
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00002669 MadeChange = true;
Simon Pilgrim471efd22016-02-20 23:17:35 +00002670 }
2671 if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, 1)) {
2672 II->setArgOperand(1, V);
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00002673 MadeChange = true;
Simon Pilgrim471efd22016-02-20 23:17:35 +00002674 }
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00002675 if (MadeChange)
2676 return II;
Simon Pilgrim471efd22016-02-20 23:17:35 +00002677 break;
2678 }
Craig Topper31cbe752018-06-27 15:57:53 +00002679 case Intrinsic::x86_avx512_cmp_pd_128:
2680 case Intrinsic::x86_avx512_cmp_pd_256:
2681 case Intrinsic::x86_avx512_cmp_pd_512:
2682 case Intrinsic::x86_avx512_cmp_ps_128:
2683 case Intrinsic::x86_avx512_cmp_ps_256:
2684 case Intrinsic::x86_avx512_cmp_ps_512: {
Michael Zuckerman16b20d22017-04-16 13:26:08 +00002685 // Folding cmp(sub(a,b),0) -> cmp(a,b) and cmp(0,sub(a,b)) -> cmp(b,a)
2686 Value *Arg0 = II->getArgOperand(0);
2687 Value *Arg1 = II->getArgOperand(1);
Sanjay Patel93e64dd2018-03-25 21:16:33 +00002688 bool Arg0IsZero = match(Arg0, m_PosZeroFP());
Michael Zuckerman16b20d22017-04-16 13:26:08 +00002689 if (Arg0IsZero)
2690 std::swap(Arg0, Arg1);
2691 Value *A, *B;
2692 // This fold requires only the NINF(not +/- inf) since inf minus
2693 // inf is nan.
2694 // NSZ(No Signed Zeros) is not needed because zeros of any sign are
2695 // equal for both compares.
2696 // NNAN is not needed because nans compare the same for both compares.
2697 // The compare intrinsic uses the above assumptions and therefore
2698 // doesn't require additional flags.
2699 if ((match(Arg0, m_OneUse(m_FSub(m_Value(A), m_Value(B)))) &&
Sanjay Patel93e64dd2018-03-25 21:16:33 +00002700 match(Arg1, m_PosZeroFP()) && isa<Instruction>(Arg0) &&
Michael Zuckerman16b20d22017-04-16 13:26:08 +00002701 cast<Instruction>(Arg0)->getFastMathFlags().noInfs())) {
2702 if (Arg0IsZero)
2703 std::swap(A, B);
2704 II->setArgOperand(0, A);
2705 II->setArgOperand(1, B);
2706 return II;
2707 }
2708 break;
2709 }
Simon Pilgrim471efd22016-02-20 23:17:35 +00002710
Craig Topper98a79932018-06-10 06:01:36 +00002711 case Intrinsic::x86_avx512_add_ps_512:
2712 case Intrinsic::x86_avx512_div_ps_512:
2713 case Intrinsic::x86_avx512_mul_ps_512:
2714 case Intrinsic::x86_avx512_sub_ps_512:
2715 case Intrinsic::x86_avx512_add_pd_512:
2716 case Intrinsic::x86_avx512_div_pd_512:
2717 case Intrinsic::x86_avx512_mul_pd_512:
2718 case Intrinsic::x86_avx512_sub_pd_512:
Craig Topper020b2282016-12-27 00:23:16 +00002719 // If the rounding mode is CUR_DIRECTION(4) we can turn these into regular
2720 // IR operations.
Craig Topper98a79932018-06-10 06:01:36 +00002721 if (auto *R = dyn_cast<ConstantInt>(II->getArgOperand(2))) {
Craig Topper020b2282016-12-27 00:23:16 +00002722 if (R->getValue() == 4) {
2723 Value *Arg0 = II->getArgOperand(0);
2724 Value *Arg1 = II->getArgOperand(1);
2725
2726 Value *V;
Sanjay Patel62f457b2019-05-06 15:35:02 +00002727 switch (IID) {
Craig Topper020b2282016-12-27 00:23:16 +00002728 default: llvm_unreachable("Case stmts out of sync!");
Craig Topper98a79932018-06-10 06:01:36 +00002729 case Intrinsic::x86_avx512_add_ps_512:
2730 case Intrinsic::x86_avx512_add_pd_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00002731 V = Builder.CreateFAdd(Arg0, Arg1);
Craig Topper020b2282016-12-27 00:23:16 +00002732 break;
Craig Topper98a79932018-06-10 06:01:36 +00002733 case Intrinsic::x86_avx512_sub_ps_512:
2734 case Intrinsic::x86_avx512_sub_pd_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00002735 V = Builder.CreateFSub(Arg0, Arg1);
Craig Topper020b2282016-12-27 00:23:16 +00002736 break;
Craig Topper98a79932018-06-10 06:01:36 +00002737 case Intrinsic::x86_avx512_mul_ps_512:
2738 case Intrinsic::x86_avx512_mul_pd_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00002739 V = Builder.CreateFMul(Arg0, Arg1);
Craig Topper020b2282016-12-27 00:23:16 +00002740 break;
Craig Topper98a79932018-06-10 06:01:36 +00002741 case Intrinsic::x86_avx512_div_ps_512:
2742 case Intrinsic::x86_avx512_div_pd_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00002743 V = Builder.CreateFDiv(Arg0, Arg1);
Craig Topper020b2282016-12-27 00:23:16 +00002744 break;
2745 }
2746
Craig Topper020b2282016-12-27 00:23:16 +00002747 return replaceInstUsesWith(*II, V);
2748 }
2749 }
2750 break;
2751
Craig Topper790d0fa2016-12-11 07:42:01 +00002752 case Intrinsic::x86_avx512_mask_add_ss_round:
2753 case Intrinsic::x86_avx512_mask_div_ss_round:
2754 case Intrinsic::x86_avx512_mask_mul_ss_round:
2755 case Intrinsic::x86_avx512_mask_sub_ss_round:
Craig Topper790d0fa2016-12-11 07:42:01 +00002756 case Intrinsic::x86_avx512_mask_add_sd_round:
2757 case Intrinsic::x86_avx512_mask_div_sd_round:
2758 case Intrinsic::x86_avx512_mask_mul_sd_round:
2759 case Intrinsic::x86_avx512_mask_sub_sd_round:
Craig Topper7b788ada2016-12-26 06:33:19 +00002760 // If the rounding mode is CUR_DIRECTION(4) we can turn these into regular
2761 // IR operations.
2762 if (auto *R = dyn_cast<ConstantInt>(II->getArgOperand(4))) {
2763 if (R->getValue() == 4) {
Craig Topper7f8540b2016-12-27 01:56:30 +00002764 // Extract the element as scalars.
2765 Value *Arg0 = II->getArgOperand(0);
2766 Value *Arg1 = II->getArgOperand(1);
Craig Topperbb4069e2017-07-07 23:16:26 +00002767 Value *LHS = Builder.CreateExtractElement(Arg0, (uint64_t)0);
2768 Value *RHS = Builder.CreateExtractElement(Arg1, (uint64_t)0);
Craig Topper7b788ada2016-12-26 06:33:19 +00002769
Craig Topper7f8540b2016-12-27 01:56:30 +00002770 Value *V;
Sanjay Patel62f457b2019-05-06 15:35:02 +00002771 switch (IID) {
Craig Topper7f8540b2016-12-27 01:56:30 +00002772 default: llvm_unreachable("Case stmts out of sync!");
2773 case Intrinsic::x86_avx512_mask_add_ss_round:
2774 case Intrinsic::x86_avx512_mask_add_sd_round:
Craig Topperbb4069e2017-07-07 23:16:26 +00002775 V = Builder.CreateFAdd(LHS, RHS);
Craig Topper7f8540b2016-12-27 01:56:30 +00002776 break;
2777 case Intrinsic::x86_avx512_mask_sub_ss_round:
2778 case Intrinsic::x86_avx512_mask_sub_sd_round:
Craig Topperbb4069e2017-07-07 23:16:26 +00002779 V = Builder.CreateFSub(LHS, RHS);
Craig Topper7f8540b2016-12-27 01:56:30 +00002780 break;
2781 case Intrinsic::x86_avx512_mask_mul_ss_round:
2782 case Intrinsic::x86_avx512_mask_mul_sd_round:
Craig Topperbb4069e2017-07-07 23:16:26 +00002783 V = Builder.CreateFMul(LHS, RHS);
Craig Topper7f8540b2016-12-27 01:56:30 +00002784 break;
2785 case Intrinsic::x86_avx512_mask_div_ss_round:
2786 case Intrinsic::x86_avx512_mask_div_sd_round:
Craig Topperbb4069e2017-07-07 23:16:26 +00002787 V = Builder.CreateFDiv(LHS, RHS);
Craig Topper7f8540b2016-12-27 01:56:30 +00002788 break;
Craig Topper7b788ada2016-12-26 06:33:19 +00002789 }
Craig Topper7f8540b2016-12-27 01:56:30 +00002790
2791 // Handle the masking aspect of the intrinsic.
Craig Topper7f8540b2016-12-27 01:56:30 +00002792 Value *Mask = II->getArgOperand(3);
Craig Topper99163632016-12-30 23:06:28 +00002793 auto *C = dyn_cast<ConstantInt>(Mask);
2794 // We don't need a select if we know the mask bit is a 1.
2795 if (!C || !C->getValue()[0]) {
2796 // Cast the mask to an i1 vector and then extract the lowest element.
Craig Topperbb4069e2017-07-07 23:16:26 +00002797 auto *MaskTy = VectorType::get(Builder.getInt1Ty(),
Craig Topper7f8540b2016-12-27 01:56:30 +00002798 cast<IntegerType>(Mask->getType())->getBitWidth());
Craig Topperbb4069e2017-07-07 23:16:26 +00002799 Mask = Builder.CreateBitCast(Mask, MaskTy);
2800 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
Craig Topper99163632016-12-30 23:06:28 +00002801 // Extract the lowest element from the passthru operand.
Craig Topperbb4069e2017-07-07 23:16:26 +00002802 Value *Passthru = Builder.CreateExtractElement(II->getArgOperand(2),
Craig Topper99163632016-12-30 23:06:28 +00002803 (uint64_t)0);
Craig Topperbb4069e2017-07-07 23:16:26 +00002804 V = Builder.CreateSelect(Mask, V, Passthru);
Craig Topper99163632016-12-30 23:06:28 +00002805 }
Craig Topper7f8540b2016-12-27 01:56:30 +00002806
2807 // Insert the result back into the original argument 0.
Craig Topperbb4069e2017-07-07 23:16:26 +00002808 V = Builder.CreateInsertElement(Arg0, V, (uint64_t)0);
Craig Topper7f8540b2016-12-27 01:56:30 +00002809
2810 return replaceInstUsesWith(*II, V);
Craig Topper7b788ada2016-12-26 06:33:19 +00002811 }
2812 }
Philip Reamesc71e9962019-01-30 19:21:11 +00002813 break;
Craig Topper7b788ada2016-12-26 06:33:19 +00002814
Mikhail Dvoretckii8393f902018-06-19 10:49:12 +00002815 case Intrinsic::x86_sse41_round_ss:
2816 case Intrinsic::x86_sse41_round_sd: {
Philip Reamesc71e9962019-01-30 19:21:11 +00002817 if (Value *V = simplifyX86round(*II, Builder))
Mikhail Dvoretckii8393f902018-06-19 10:49:12 +00002818 return replaceInstUsesWith(*II, V);
2819 break;
2820 }
Craig Topperac75bca2016-12-13 07:45:45 +00002821
Simon Pilgrima3a72b42015-08-10 20:21:15 +00002822 // Constant fold ashr( <A x Bi>, Ci ).
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002823 // Constant fold lshr( <A x Bi>, Ci ).
2824 // Constant fold shl( <A x Bi>, Ci ).
Simon Pilgrima3a72b42015-08-10 20:21:15 +00002825 case Intrinsic::x86_sse2_psrai_d:
2826 case Intrinsic::x86_sse2_psrai_w:
Simon Pilgrima3a72b42015-08-10 20:21:15 +00002827 case Intrinsic::x86_avx2_psrai_d:
2828 case Intrinsic::x86_avx2_psrai_w:
Craig Topper8b831cb2016-11-13 01:51:55 +00002829 case Intrinsic::x86_avx512_psrai_q_128:
2830 case Intrinsic::x86_avx512_psrai_q_256:
2831 case Intrinsic::x86_avx512_psrai_d_512:
2832 case Intrinsic::x86_avx512_psrai_q_512:
2833 case Intrinsic::x86_avx512_psrai_w_512:
Simon Pilgrim18617d12015-08-05 08:18:00 +00002834 case Intrinsic::x86_sse2_psrli_d:
2835 case Intrinsic::x86_sse2_psrli_q:
2836 case Intrinsic::x86_sse2_psrli_w:
Simon Pilgrim18617d12015-08-05 08:18:00 +00002837 case Intrinsic::x86_avx2_psrli_d:
2838 case Intrinsic::x86_avx2_psrli_q:
2839 case Intrinsic::x86_avx2_psrli_w:
Craig Topper8b831cb2016-11-13 01:51:55 +00002840 case Intrinsic::x86_avx512_psrli_d_512:
2841 case Intrinsic::x86_avx512_psrli_q_512:
2842 case Intrinsic::x86_avx512_psrli_w_512:
Michael J. Spencerdee4b2c2014-04-24 00:58:18 +00002843 case Intrinsic::x86_sse2_pslli_d:
2844 case Intrinsic::x86_sse2_pslli_q:
2845 case Intrinsic::x86_sse2_pslli_w:
Simon Pilgrim18617d12015-08-05 08:18:00 +00002846 case Intrinsic::x86_avx2_pslli_d:
2847 case Intrinsic::x86_avx2_pslli_q:
2848 case Intrinsic::x86_avx2_pslli_w:
Craig Topper8b831cb2016-11-13 01:51:55 +00002849 case Intrinsic::x86_avx512_pslli_d_512:
2850 case Intrinsic::x86_avx512_pslli_q_512:
2851 case Intrinsic::x86_avx512_pslli_w_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00002852 if (Value *V = simplifyX86immShift(*II, Builder))
Sanjay Patel4b198802016-02-01 22:23:39 +00002853 return replaceInstUsesWith(*II, V);
Simon Pilgrim18617d12015-08-05 08:18:00 +00002854 break;
2855
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002856 case Intrinsic::x86_sse2_psra_d:
2857 case Intrinsic::x86_sse2_psra_w:
2858 case Intrinsic::x86_avx2_psra_d:
2859 case Intrinsic::x86_avx2_psra_w:
Craig Topper8b831cb2016-11-13 01:51:55 +00002860 case Intrinsic::x86_avx512_psra_q_128:
2861 case Intrinsic::x86_avx512_psra_q_256:
2862 case Intrinsic::x86_avx512_psra_d_512:
2863 case Intrinsic::x86_avx512_psra_q_512:
2864 case Intrinsic::x86_avx512_psra_w_512:
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002865 case Intrinsic::x86_sse2_psrl_d:
2866 case Intrinsic::x86_sse2_psrl_q:
2867 case Intrinsic::x86_sse2_psrl_w:
2868 case Intrinsic::x86_avx2_psrl_d:
2869 case Intrinsic::x86_avx2_psrl_q:
2870 case Intrinsic::x86_avx2_psrl_w:
Craig Topper8b831cb2016-11-13 01:51:55 +00002871 case Intrinsic::x86_avx512_psrl_d_512:
2872 case Intrinsic::x86_avx512_psrl_q_512:
2873 case Intrinsic::x86_avx512_psrl_w_512:
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002874 case Intrinsic::x86_sse2_psll_d:
2875 case Intrinsic::x86_sse2_psll_q:
2876 case Intrinsic::x86_sse2_psll_w:
2877 case Intrinsic::x86_avx2_psll_d:
2878 case Intrinsic::x86_avx2_psll_q:
Craig Topper8b831cb2016-11-13 01:51:55 +00002879 case Intrinsic::x86_avx2_psll_w:
2880 case Intrinsic::x86_avx512_psll_d_512:
2881 case Intrinsic::x86_avx512_psll_q_512:
2882 case Intrinsic::x86_avx512_psll_w_512: {
Craig Topperbb4069e2017-07-07 23:16:26 +00002883 if (Value *V = simplifyX86immShift(*II, Builder))
Sanjay Patel4b198802016-02-01 22:23:39 +00002884 return replaceInstUsesWith(*II, V);
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002885
2886 // SSE2/AVX2 uses only the first 64-bits of the 128-bit vector
2887 // operand to compute the shift amount.
Simon Pilgrim996725e2015-09-19 11:41:53 +00002888 Value *Arg1 = II->getArgOperand(1);
2889 assert(Arg1->getType()->getPrimitiveSizeInBits() == 128 &&
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002890 "Unexpected packed shift size");
Simon Pilgrim996725e2015-09-19 11:41:53 +00002891 unsigned VWidth = Arg1->getType()->getVectorNumElements();
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002892
Simon Pilgrim996725e2015-09-19 11:41:53 +00002893 if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, VWidth / 2)) {
Simon Pilgrimbecd5e82015-08-13 07:39:03 +00002894 II->setArgOperand(1, V);
2895 return II;
2896 }
2897 break;
2898 }
2899
Simon Pilgrimdb9893f2016-06-07 10:27:15 +00002900 case Intrinsic::x86_avx2_psllv_d:
2901 case Intrinsic::x86_avx2_psllv_d_256:
2902 case Intrinsic::x86_avx2_psllv_q:
2903 case Intrinsic::x86_avx2_psllv_q_256:
Craig Topperb4173a52016-11-13 07:26:19 +00002904 case Intrinsic::x86_avx512_psllv_d_512:
2905 case Intrinsic::x86_avx512_psllv_q_512:
Craig Topper1de753f2016-11-18 06:04:33 +00002906 case Intrinsic::x86_avx512_psllv_w_128:
2907 case Intrinsic::x86_avx512_psllv_w_256:
2908 case Intrinsic::x86_avx512_psllv_w_512:
Simon Pilgrimdb9893f2016-06-07 10:27:15 +00002909 case Intrinsic::x86_avx2_psrav_d:
2910 case Intrinsic::x86_avx2_psrav_d_256:
Craig Topperb4173a52016-11-13 07:26:19 +00002911 case Intrinsic::x86_avx512_psrav_q_128:
2912 case Intrinsic::x86_avx512_psrav_q_256:
2913 case Intrinsic::x86_avx512_psrav_d_512:
2914 case Intrinsic::x86_avx512_psrav_q_512:
Craig Topper1de753f2016-11-18 06:04:33 +00002915 case Intrinsic::x86_avx512_psrav_w_128:
2916 case Intrinsic::x86_avx512_psrav_w_256:
2917 case Intrinsic::x86_avx512_psrav_w_512:
Simon Pilgrimdb9893f2016-06-07 10:27:15 +00002918 case Intrinsic::x86_avx2_psrlv_d:
2919 case Intrinsic::x86_avx2_psrlv_d_256:
2920 case Intrinsic::x86_avx2_psrlv_q:
2921 case Intrinsic::x86_avx2_psrlv_q_256:
Craig Topperb4173a52016-11-13 07:26:19 +00002922 case Intrinsic::x86_avx512_psrlv_d_512:
2923 case Intrinsic::x86_avx512_psrlv_q_512:
Craig Topper1de753f2016-11-18 06:04:33 +00002924 case Intrinsic::x86_avx512_psrlv_w_128:
2925 case Intrinsic::x86_avx512_psrlv_w_256:
2926 case Intrinsic::x86_avx512_psrlv_w_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00002927 if (Value *V = simplifyX86varShift(*II, Builder))
Simon Pilgrimdb9893f2016-06-07 10:27:15 +00002928 return replaceInstUsesWith(*II, V);
2929 break;
2930
Simon Pilgrim6f6b2792017-01-25 14:37:24 +00002931 case Intrinsic::x86_sse2_packssdw_128:
2932 case Intrinsic::x86_sse2_packsswb_128:
2933 case Intrinsic::x86_avx2_packssdw:
2934 case Intrinsic::x86_avx2_packsswb:
Craig Topper3731f4d2017-02-16 07:35:23 +00002935 case Intrinsic::x86_avx512_packssdw_512:
2936 case Intrinsic::x86_avx512_packsswb_512:
Simon Pilgrim55f14da2019-04-24 16:53:17 +00002937 if (Value *V = simplifyX86pack(*II, Builder, true))
Simon Pilgrim6f6b2792017-01-25 14:37:24 +00002938 return replaceInstUsesWith(*II, V);
2939 break;
2940
2941 case Intrinsic::x86_sse2_packuswb_128:
2942 case Intrinsic::x86_sse41_packusdw:
2943 case Intrinsic::x86_avx2_packusdw:
2944 case Intrinsic::x86_avx2_packuswb:
Craig Topper3731f4d2017-02-16 07:35:23 +00002945 case Intrinsic::x86_avx512_packusdw_512:
2946 case Intrinsic::x86_avx512_packuswb_512:
Simon Pilgrim55f14da2019-04-24 16:53:17 +00002947 if (Value *V = simplifyX86pack(*II, Builder, false))
Simon Pilgrim6f6b2792017-01-25 14:37:24 +00002948 return replaceInstUsesWith(*II, V);
2949 break;
2950
Craig Topper911025b2018-05-13 21:56:32 +00002951 case Intrinsic::x86_pclmulqdq:
2952 case Intrinsic::x86_pclmulqdq_256:
2953 case Intrinsic::x86_pclmulqdq_512: {
Craig Topperb6122122017-01-26 05:17:13 +00002954 if (auto *C = dyn_cast<ConstantInt>(II->getArgOperand(2))) {
2955 unsigned Imm = C->getZExtValue();
2956
2957 bool MadeChange = false;
2958 Value *Arg0 = II->getArgOperand(0);
2959 Value *Arg1 = II->getArgOperand(1);
2960 unsigned VWidth = Arg0->getType()->getVectorNumElements();
Craig Topperb6122122017-01-26 05:17:13 +00002961
2962 APInt UndefElts1(VWidth, 0);
Craig Topper911025b2018-05-13 21:56:32 +00002963 APInt DemandedElts1 = APInt::getSplat(VWidth,
2964 APInt(2, (Imm & 0x01) ? 2 : 1));
2965 if (Value *V = SimplifyDemandedVectorElts(Arg0, DemandedElts1,
Craig Topperb6122122017-01-26 05:17:13 +00002966 UndefElts1)) {
2967 II->setArgOperand(0, V);
2968 MadeChange = true;
2969 }
2970
2971 APInt UndefElts2(VWidth, 0);
Craig Topper911025b2018-05-13 21:56:32 +00002972 APInt DemandedElts2 = APInt::getSplat(VWidth,
2973 APInt(2, (Imm & 0x10) ? 2 : 1));
2974 if (Value *V = SimplifyDemandedVectorElts(Arg1, DemandedElts2,
Craig Topperb6122122017-01-26 05:17:13 +00002975 UndefElts2)) {
2976 II->setArgOperand(1, V);
2977 MadeChange = true;
2978 }
2979
Craig Topper911025b2018-05-13 21:56:32 +00002980 // If either input elements are undef, the result is zero.
2981 if (DemandedElts1.isSubsetOf(UndefElts1) ||
2982 DemandedElts2.isSubsetOf(UndefElts2))
Craig Topperb6122122017-01-26 05:17:13 +00002983 return replaceInstUsesWith(*II,
2984 ConstantAggregateZero::get(II->getType()));
2985
2986 if (MadeChange)
2987 return II;
2988 }
2989 break;
2990 }
2991
Sanjay Patelc86867c2015-04-16 17:52:13 +00002992 case Intrinsic::x86_sse41_insertps:
Craig Topperbb4069e2017-07-07 23:16:26 +00002993 if (Value *V = simplifyX86insertps(*II, Builder))
Sanjay Patel4b198802016-02-01 22:23:39 +00002994 return replaceInstUsesWith(*II, V);
Sanjay Patelc86867c2015-04-16 17:52:13 +00002995 break;
Simon Pilgrim54fcd622015-07-25 20:41:00 +00002996
Simon Pilgrim61116dd2015-09-17 20:32:45 +00002997 case Intrinsic::x86_sse4a_extrq: {
Simon Pilgrim61116dd2015-09-17 20:32:45 +00002998 Value *Op0 = II->getArgOperand(0);
2999 Value *Op1 = II->getArgOperand(1);
3000 unsigned VWidth0 = Op0->getType()->getVectorNumElements();
3001 unsigned VWidth1 = Op1->getType()->getVectorNumElements();
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003002 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
3003 Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
3004 VWidth1 == 16 && "Unexpected operand sizes");
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003005
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003006 // See if we're dealing with constant values.
3007 Constant *C1 = dyn_cast<Constant>(Op1);
3008 ConstantInt *CILength =
Andrea Di Biagio8df5b9c2016-09-07 12:03:03 +00003009 C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)0))
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003010 : nullptr;
3011 ConstantInt *CIIndex =
Andrea Di Biagio8df5b9c2016-09-07 12:03:03 +00003012 C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)1))
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003013 : nullptr;
3014
3015 // Attempt to simplify to a constant, shuffle vector or EXTRQI call.
Craig Topperbb4069e2017-07-07 23:16:26 +00003016 if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, Builder))
Sanjay Patel4b198802016-02-01 22:23:39 +00003017 return replaceInstUsesWith(*II, V);
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003018
3019 // EXTRQ only uses the lowest 64-bits of the first 128-bit vector
3020 // operands and the lowest 16-bits of the second.
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003021 bool MadeChange = false;
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003022 if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
3023 II->setArgOperand(0, V);
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003024 MadeChange = true;
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003025 }
3026 if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 2)) {
3027 II->setArgOperand(1, V);
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003028 MadeChange = true;
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003029 }
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003030 if (MadeChange)
3031 return II;
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003032 break;
3033 }
3034
3035 case Intrinsic::x86_sse4a_extrqi: {
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003036 // EXTRQI: Extract Length bits starting from Index. Zero pad the remaining
3037 // bits of the lower 64-bits. The upper 64-bits are undefined.
3038 Value *Op0 = II->getArgOperand(0);
3039 unsigned VWidth = Op0->getType()->getVectorNumElements();
3040 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
3041 "Unexpected operand size");
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003042
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003043 // See if we're dealing with constant values.
3044 ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(1));
3045 ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(2));
3046
3047 // Attempt to simplify to a constant or shuffle vector.
Craig Topperbb4069e2017-07-07 23:16:26 +00003048 if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, Builder))
Sanjay Patel4b198802016-02-01 22:23:39 +00003049 return replaceInstUsesWith(*II, V);
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003050
3051 // EXTRQI only uses the lowest 64-bits of the first 128-bit vector
3052 // operand.
3053 if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003054 II->setArgOperand(0, V);
3055 return II;
3056 }
3057 break;
3058 }
3059
3060 case Intrinsic::x86_sse4a_insertq: {
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003061 Value *Op0 = II->getArgOperand(0);
3062 Value *Op1 = II->getArgOperand(1);
3063 unsigned VWidth = Op0->getType()->getVectorNumElements();
3064 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
3065 Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
3066 Op1->getType()->getVectorNumElements() == 2 &&
3067 "Unexpected operand size");
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003068
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003069 // See if we're dealing with constant values.
3070 Constant *C1 = dyn_cast<Constant>(Op1);
3071 ConstantInt *CI11 =
Andrea Di Biagiof3fd3162016-09-07 12:47:53 +00003072 C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)1))
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003073 : nullptr;
3074
3075 // Attempt to simplify to a constant, shuffle vector or INSERTQI call.
3076 if (CI11) {
Benjamin Kramer46e38f32016-06-08 10:01:20 +00003077 const APInt &V11 = CI11->getValue();
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003078 APInt Len = V11.zextOrTrunc(6);
3079 APInt Idx = V11.lshr(8).zextOrTrunc(6);
Craig Topperbb4069e2017-07-07 23:16:26 +00003080 if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, Builder))
Sanjay Patel4b198802016-02-01 22:23:39 +00003081 return replaceInstUsesWith(*II, V);
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003082 }
3083
3084 // INSERTQ only uses the lowest 64-bits of the first 128-bit vector
3085 // operand.
3086 if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003087 II->setArgOperand(0, V);
3088 return II;
3089 }
3090 break;
3091 }
3092
Filipe Cabecinhas1a805952014-04-24 00:38:14 +00003093 case Intrinsic::x86_sse4a_insertqi: {
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003094 // INSERTQI: Extract lowest Length bits from lower half of second source and
3095 // insert over first source starting at Index bit. The upper 64-bits are
3096 // undefined.
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003097 Value *Op0 = II->getArgOperand(0);
3098 Value *Op1 = II->getArgOperand(1);
3099 unsigned VWidth0 = Op0->getType()->getVectorNumElements();
3100 unsigned VWidth1 = Op1->getType()->getVectorNumElements();
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003101 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
3102 Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
3103 VWidth1 == 2 && "Unexpected operand sizes");
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003104
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003105 // See if we're dealing with constant values.
3106 ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(2));
3107 ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(3));
3108
3109 // Attempt to simplify to a constant or shuffle vector.
3110 if (CILength && CIIndex) {
3111 APInt Len = CILength->getValue().zextOrTrunc(6);
3112 APInt Idx = CIIndex->getValue().zextOrTrunc(6);
Craig Topperbb4069e2017-07-07 23:16:26 +00003113 if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, Builder))
Sanjay Patel4b198802016-02-01 22:23:39 +00003114 return replaceInstUsesWith(*II, V);
Simon Pilgrim216b1bf2015-10-17 11:40:05 +00003115 }
3116
3117 // INSERTQI only uses the lowest 64-bits of the first two 128-bit vector
3118 // operands.
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003119 bool MadeChange = false;
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003120 if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
3121 II->setArgOperand(0, V);
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003122 MadeChange = true;
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003123 }
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003124 if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 1)) {
3125 II->setArgOperand(1, V);
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003126 MadeChange = true;
Simon Pilgrim61116dd2015-09-17 20:32:45 +00003127 }
Simon Pilgrim1c9a9f22016-04-24 17:57:27 +00003128 if (MadeChange)
3129 return II;
Filipe Cabecinhas1a805952014-04-24 00:38:14 +00003130 break;
3131 }
3132
Filipe Cabecinhas82ac07c2014-05-27 03:42:20 +00003133 case Intrinsic::x86_sse41_pblendvb:
3134 case Intrinsic::x86_sse41_blendvps:
3135 case Intrinsic::x86_sse41_blendvpd:
3136 case Intrinsic::x86_avx_blendv_ps_256:
3137 case Intrinsic::x86_avx_blendv_pd_256:
3138 case Intrinsic::x86_avx2_pblendvb: {
Sanjay Patel296d35a2018-09-15 14:25:44 +00003139 // fold (blend A, A, Mask) -> A
Simon Pilgrim8c049d52015-08-12 08:08:56 +00003140 Value *Op0 = II->getArgOperand(0);
3141 Value *Op1 = II->getArgOperand(1);
Filipe Cabecinhas82ac07c2014-05-27 03:42:20 +00003142 Value *Mask = II->getArgOperand(2);
Simon Pilgrim8c049d52015-08-12 08:08:56 +00003143 if (Op0 == Op1)
Sanjay Patel4b198802016-02-01 22:23:39 +00003144 return replaceInstUsesWith(CI, Op0);
Simon Pilgrim8c049d52015-08-12 08:08:56 +00003145
3146 // Zero Mask - select 1st argument.
Simon Pilgrim93f59f52015-08-12 08:23:36 +00003147 if (isa<ConstantAggregateZero>(Mask))
Sanjay Patel4b198802016-02-01 22:23:39 +00003148 return replaceInstUsesWith(CI, Op0);
Simon Pilgrim8c049d52015-08-12 08:08:56 +00003149
3150 // Constant Mask - select 1st/2nd argument lane based on top bit of mask.
Sanjay Patel368ac5d2016-02-21 17:29:33 +00003151 if (auto *ConstantMask = dyn_cast<ConstantDataVector>(Mask)) {
3152 Constant *NewSelector = getNegativeIsTrueBoolVec(ConstantMask);
Simon Pilgrim8c049d52015-08-12 08:08:56 +00003153 return SelectInst::Create(NewSelector, Op1, Op0, "blendv");
Filipe Cabecinhas82ac07c2014-05-27 03:42:20 +00003154 }
Sanjay Patel296d35a2018-09-15 14:25:44 +00003155
3156 // Convert to a vector select if we can bypass casts and find a boolean
3157 // vector condition value.
3158 Value *BoolVec;
Sanjay Patel09e02fb2018-09-22 14:43:55 +00003159 Mask = peekThroughBitcast(Mask);
3160 if (match(Mask, m_SExt(m_Value(BoolVec))) &&
3161 BoolVec->getType()->isVectorTy() &&
3162 BoolVec->getType()->getScalarSizeInBits() == 1) {
3163 assert(Mask->getType()->getPrimitiveSizeInBits() ==
3164 II->getType()->getPrimitiveSizeInBits() &&
3165 "Not expecting mask and operands with different sizes");
3166
3167 unsigned NumMaskElts = Mask->getType()->getVectorNumElements();
3168 unsigned NumOperandElts = II->getType()->getVectorNumElements();
3169 if (NumMaskElts == NumOperandElts)
Sanjay Patel296d35a2018-09-15 14:25:44 +00003170 return SelectInst::Create(BoolVec, Op1, Op0);
Sanjay Patel09e02fb2018-09-22 14:43:55 +00003171
3172 // If the mask has less elements than the operands, each mask bit maps to
3173 // multiple elements of the operands. Bitcast back and forth.
3174 if (NumMaskElts < NumOperandElts) {
3175 Value *CastOp0 = Builder.CreateBitCast(Op0, Mask->getType());
3176 Value *CastOp1 = Builder.CreateBitCast(Op1, Mask->getType());
3177 Value *Sel = Builder.CreateSelect(BoolVec, CastOp1, CastOp0);
3178 return new BitCastInst(Sel, II->getType());
3179 }
Sanjay Patel296d35a2018-09-15 14:25:44 +00003180 }
3181
Simon Pilgrim8c049d52015-08-12 08:08:56 +00003182 break;
Filipe Cabecinhas82ac07c2014-05-27 03:42:20 +00003183 }
3184
Andrea Di Biagio0594e2a2015-09-30 16:44:39 +00003185 case Intrinsic::x86_ssse3_pshuf_b_128:
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00003186 case Intrinsic::x86_avx2_pshuf_b:
Simon Pilgrima22c3a12017-01-18 13:44:04 +00003187 case Intrinsic::x86_avx512_pshuf_b_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00003188 if (Value *V = simplifyX86pshufb(*II, Builder))
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00003189 return replaceInstUsesWith(*II, V);
3190 break;
Andrea Di Biagio0594e2a2015-09-30 16:44:39 +00003191
Rafael Espindolabad3f772014-04-21 22:06:04 +00003192 case Intrinsic::x86_avx_vpermilvar_ps:
3193 case Intrinsic::x86_avx_vpermilvar_ps_256:
Craig Topper58917f32016-12-11 01:59:36 +00003194 case Intrinsic::x86_avx512_vpermilvar_ps_512:
Rafael Espindolabad3f772014-04-21 22:06:04 +00003195 case Intrinsic::x86_avx_vpermilvar_pd:
Simon Pilgrim2f6097d2016-04-24 17:23:46 +00003196 case Intrinsic::x86_avx_vpermilvar_pd_256:
Simon Pilgrima22c3a12017-01-18 13:44:04 +00003197 case Intrinsic::x86_avx512_vpermilvar_pd_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00003198 if (Value *V = simplifyX86vpermilvar(*II, Builder))
Simon Pilgrim2f6097d2016-04-24 17:23:46 +00003199 return replaceInstUsesWith(*II, V);
3200 break;
Rafael Espindolabad3f772014-04-21 22:06:04 +00003201
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00003202 case Intrinsic::x86_avx2_permd:
3203 case Intrinsic::x86_avx2_permps:
Craig Toppere4c045b2018-05-20 23:34:04 +00003204 case Intrinsic::x86_avx512_permvar_df_256:
3205 case Intrinsic::x86_avx512_permvar_df_512:
3206 case Intrinsic::x86_avx512_permvar_di_256:
3207 case Intrinsic::x86_avx512_permvar_di_512:
3208 case Intrinsic::x86_avx512_permvar_hi_128:
3209 case Intrinsic::x86_avx512_permvar_hi_256:
3210 case Intrinsic::x86_avx512_permvar_hi_512:
3211 case Intrinsic::x86_avx512_permvar_qi_128:
3212 case Intrinsic::x86_avx512_permvar_qi_256:
3213 case Intrinsic::x86_avx512_permvar_qi_512:
3214 case Intrinsic::x86_avx512_permvar_sf_512:
3215 case Intrinsic::x86_avx512_permvar_si_512:
Craig Topperbb4069e2017-07-07 23:16:26 +00003216 if (Value *V = simplifyX86vpermv(*II, Builder))
Simon Pilgrim8cddf8b2016-05-01 16:41:22 +00003217 return replaceInstUsesWith(*II, V);
3218 break;
3219
Sanjay Patel98a71502016-02-29 23:16:48 +00003220 case Intrinsic::x86_avx_maskload_ps:
Sanjay Patel6f2c01f2016-02-29 23:59:00 +00003221 case Intrinsic::x86_avx_maskload_pd:
3222 case Intrinsic::x86_avx_maskload_ps_256:
3223 case Intrinsic::x86_avx_maskload_pd_256:
3224 case Intrinsic::x86_avx2_maskload_d:
3225 case Intrinsic::x86_avx2_maskload_q:
3226 case Intrinsic::x86_avx2_maskload_d_256:
3227 case Intrinsic::x86_avx2_maskload_q_256:
Sanjay Patel98a71502016-02-29 23:16:48 +00003228 if (Instruction *I = simplifyX86MaskedLoad(*II, *this))
3229 return I;
3230 break;
3231
Sanjay Patelc4acbae2016-03-12 15:16:59 +00003232 case Intrinsic::x86_sse2_maskmov_dqu:
Sanjay Patel1ace9932016-02-26 21:04:14 +00003233 case Intrinsic::x86_avx_maskstore_ps:
3234 case Intrinsic::x86_avx_maskstore_pd:
3235 case Intrinsic::x86_avx_maskstore_ps_256:
3236 case Intrinsic::x86_avx_maskstore_pd_256:
Sanjay Patelfc7e7eb2016-02-26 21:51:44 +00003237 case Intrinsic::x86_avx2_maskstore_d:
3238 case Intrinsic::x86_avx2_maskstore_q:
3239 case Intrinsic::x86_avx2_maskstore_d_256:
3240 case Intrinsic::x86_avx2_maskstore_q_256:
Sanjay Patel1ace9932016-02-26 21:04:14 +00003241 if (simplifyX86MaskedStore(*II, *this))
3242 return nullptr;
3243 break;
3244
Sanjay Patelbe23a912019-02-01 14:14:47 +00003245 case Intrinsic::x86_addcarry_32:
3246 case Intrinsic::x86_addcarry_64:
3247 if (Value *V = simplifyX86addcarry(*II, Builder))
3248 return replaceInstUsesWith(*II, V);
3249 break;
3250
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003251 case Intrinsic::ppc_altivec_vperm:
3252 // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
Bill Schmidta1184632014-06-05 19:46:04 +00003253 // Note that ppc_altivec_vperm has a big-endian bias, so when creating
3254 // a vectorshuffle for little endian, we must undo the transformation
3255 // performed on vec_perm in altivec.h. That is, we must complement
3256 // the permutation mask with respect to 31 and reverse the order of
3257 // V1 and V2.
Chris Lattner0256be92012-01-27 03:08:05 +00003258 if (Constant *Mask = dyn_cast<Constant>(II->getArgOperand(2))) {
3259 assert(Mask->getType()->getVectorNumElements() == 16 &&
3260 "Bad type for intrinsic!");
Jim Grosbach7815f562012-02-03 00:07:04 +00003261
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003262 // Check that all of the elements are integer constants or undefs.
3263 bool AllEltsOk = true;
3264 for (unsigned i = 0; i != 16; ++i) {
Chris Lattner0256be92012-01-27 03:08:05 +00003265 Constant *Elt = Mask->getAggregateElement(i);
Craig Topperf40110f2014-04-25 05:29:35 +00003266 if (!Elt || !(isa<ConstantInt>(Elt) || isa<UndefValue>(Elt))) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003267 AllEltsOk = false;
3268 break;
3269 }
3270 }
Jim Grosbach7815f562012-02-03 00:07:04 +00003271
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003272 if (AllEltsOk) {
3273 // Cast the input vectors to byte vectors.
Craig Topperbb4069e2017-07-07 23:16:26 +00003274 Value *Op0 = Builder.CreateBitCast(II->getArgOperand(0),
3275 Mask->getType());
3276 Value *Op1 = Builder.CreateBitCast(II->getArgOperand(1),
3277 Mask->getType());
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003278 Value *Result = UndefValue::get(Op0->getType());
Jim Grosbach7815f562012-02-03 00:07:04 +00003279
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003280 // Only extract each element once.
3281 Value *ExtractedElts[32];
3282 memset(ExtractedElts, 0, sizeof(ExtractedElts));
Jim Grosbach7815f562012-02-03 00:07:04 +00003283
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003284 for (unsigned i = 0; i != 16; ++i) {
Chris Lattner0256be92012-01-27 03:08:05 +00003285 if (isa<UndefValue>(Mask->getAggregateElement(i)))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003286 continue;
Jim Grosbach7815f562012-02-03 00:07:04 +00003287 unsigned Idx =
Chris Lattner0256be92012-01-27 03:08:05 +00003288 cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003289 Idx &= 31; // Match the hardware behavior.
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003290 if (DL.isLittleEndian())
Bill Schmidta1184632014-06-05 19:46:04 +00003291 Idx = 31 - Idx;
Jim Grosbach7815f562012-02-03 00:07:04 +00003292
Craig Topperf40110f2014-04-25 05:29:35 +00003293 if (!ExtractedElts[Idx]) {
Mehdi Aminia28d91d2015-03-10 02:37:25 +00003294 Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0;
3295 Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1;
Jim Grosbach7815f562012-02-03 00:07:04 +00003296 ExtractedElts[Idx] =
Craig Topperbb4069e2017-07-07 23:16:26 +00003297 Builder.CreateExtractElement(Idx < 16 ? Op0ToUse : Op1ToUse,
3298 Builder.getInt32(Idx&15));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003299 }
Jim Grosbach7815f562012-02-03 00:07:04 +00003300
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003301 // Insert this value into the result vector.
Craig Topperbb4069e2017-07-07 23:16:26 +00003302 Result = Builder.CreateInsertElement(Result, ExtractedElts[Idx],
3303 Builder.getInt32(i));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003304 }
3305 return CastInst::Create(Instruction::BitCast, Result, CI.getType());
3306 }
3307 }
3308 break;
3309
Alexandros Lamprineas61f0ba12018-05-31 12:19:18 +00003310 case Intrinsic::arm_neon_vld1: {
3311 unsigned MemAlign = getKnownAlignment(II->getArgOperand(0),
3312 DL, II, &AC, &DT);
3313 if (Value *V = simplifyNeonVld1(*II, MemAlign, Builder))
3314 return replaceInstUsesWith(*II, V);
3315 break;
3316 }
3317
Bob Wilsona4e231c2010-10-22 21:41:48 +00003318 case Intrinsic::arm_neon_vld2:
3319 case Intrinsic::arm_neon_vld3:
3320 case Intrinsic::arm_neon_vld4:
3321 case Intrinsic::arm_neon_vld2lane:
3322 case Intrinsic::arm_neon_vld3lane:
3323 case Intrinsic::arm_neon_vld4lane:
3324 case Intrinsic::arm_neon_vst1:
3325 case Intrinsic::arm_neon_vst2:
3326 case Intrinsic::arm_neon_vst3:
3327 case Intrinsic::arm_neon_vst4:
3328 case Intrinsic::arm_neon_vst2lane:
3329 case Intrinsic::arm_neon_vst3lane:
3330 case Intrinsic::arm_neon_vst4lane: {
Justin Bogner99798402016-08-05 01:06:44 +00003331 unsigned MemAlign =
Daniel Jasperaec2fa32016-12-19 08:22:17 +00003332 getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);
Bob Wilsona4e231c2010-10-22 21:41:48 +00003333 unsigned AlignArg = II->getNumArgOperands() - 1;
3334 ConstantInt *IntrAlign = dyn_cast<ConstantInt>(II->getArgOperand(AlignArg));
3335 if (IntrAlign && IntrAlign->getZExtValue() < MemAlign) {
3336 II->setArgOperand(AlignArg,
3337 ConstantInt::get(Type::getInt32Ty(II->getContext()),
3338 MemAlign, false));
3339 return II;
3340 }
3341 break;
3342 }
3343
Alexandros Lamprineas52457d32018-05-30 14:38:50 +00003344 case Intrinsic::arm_neon_vtbl1:
3345 case Intrinsic::aarch64_neon_tbl1:
3346 if (Value *V = simplifyNeonTbl1(*II, Builder))
3347 return replaceInstUsesWith(*II, V);
3348 break;
3349
Lang Hames3a90fab2012-05-01 00:20:38 +00003350 case Intrinsic::arm_neon_vmulls:
Tim Northover00ed9962014-03-29 10:18:08 +00003351 case Intrinsic::arm_neon_vmullu:
Tim Northover3b0846e2014-05-24 12:50:23 +00003352 case Intrinsic::aarch64_neon_smull:
3353 case Intrinsic::aarch64_neon_umull: {
Lang Hames3a90fab2012-05-01 00:20:38 +00003354 Value *Arg0 = II->getArgOperand(0);
3355 Value *Arg1 = II->getArgOperand(1);
3356
3357 // Handle mul by zero first:
3358 if (isa<ConstantAggregateZero>(Arg0) || isa<ConstantAggregateZero>(Arg1)) {
Sanjay Patel4b198802016-02-01 22:23:39 +00003359 return replaceInstUsesWith(CI, ConstantAggregateZero::get(II->getType()));
Lang Hames3a90fab2012-05-01 00:20:38 +00003360 }
3361
3362 // Check for constant LHS & RHS - in this case we just simplify.
Sanjay Patel62f457b2019-05-06 15:35:02 +00003363 bool Zext = (IID == Intrinsic::arm_neon_vmullu ||
3364 IID == Intrinsic::aarch64_neon_umull);
Lang Hames3a90fab2012-05-01 00:20:38 +00003365 VectorType *NewVT = cast<VectorType>(II->getType());
Benjamin Kramer92040952014-02-13 18:23:24 +00003366 if (Constant *CV0 = dyn_cast<Constant>(Arg0)) {
3367 if (Constant *CV1 = dyn_cast<Constant>(Arg1)) {
3368 CV0 = ConstantExpr::getIntegerCast(CV0, NewVT, /*isSigned=*/!Zext);
3369 CV1 = ConstantExpr::getIntegerCast(CV1, NewVT, /*isSigned=*/!Zext);
3370
Sanjay Patel4b198802016-02-01 22:23:39 +00003371 return replaceInstUsesWith(CI, ConstantExpr::getMul(CV0, CV1));
Lang Hames3a90fab2012-05-01 00:20:38 +00003372 }
3373
Alp Tokercb402912014-01-24 17:20:08 +00003374 // Couldn't simplify - canonicalize constant to the RHS.
Lang Hames3a90fab2012-05-01 00:20:38 +00003375 std::swap(Arg0, Arg1);
3376 }
3377
3378 // Handle mul by one:
Benjamin Kramer92040952014-02-13 18:23:24 +00003379 if (Constant *CV1 = dyn_cast<Constant>(Arg1))
Lang Hames3a90fab2012-05-01 00:20:38 +00003380 if (ConstantInt *Splat =
Benjamin Kramer92040952014-02-13 18:23:24 +00003381 dyn_cast_or_null<ConstantInt>(CV1->getSplatValue()))
3382 if (Splat->isOne())
3383 return CastInst::CreateIntegerCast(Arg0, II->getType(),
3384 /*isSigned=*/!Zext);
Lang Hames3a90fab2012-05-01 00:20:38 +00003385
3386 break;
3387 }
Chad Rosier274d72f2018-05-24 15:26:42 +00003388 case Intrinsic::arm_neon_aesd:
3389 case Intrinsic::arm_neon_aese:
3390 case Intrinsic::aarch64_crypto_aesd:
3391 case Intrinsic::aarch64_crypto_aese: {
3392 Value *DataArg = II->getArgOperand(0);
3393 Value *KeyArg = II->getArgOperand(1);
3394
3395 // Try to use the builtin XOR in AESE and AESD to eliminate a prior XOR
3396 Value *Data, *Key;
3397 if (match(KeyArg, m_ZeroInt()) &&
3398 match(DataArg, m_Xor(m_Value(Data), m_Value(Key)))) {
3399 II->setArgOperand(0, Data);
3400 II->setArgOperand(1, Key);
3401 return II;
3402 }
3403 break;
3404 }
Matt Arsenaultbef34e22016-01-22 21:30:34 +00003405 case Intrinsic::amdgcn_rcp: {
Matt Arsenault4c7795d2017-03-24 19:04:57 +00003406 Value *Src = II->getArgOperand(0);
3407
3408 // TODO: Move to ConstantFolding/InstSimplify?
3409 if (isa<UndefValue>(Src))
3410 return replaceInstUsesWith(CI, Src);
3411
3412 if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
Matt Arsenaulta0050b02014-06-19 01:19:19 +00003413 const APFloat &ArgVal = C->getValueAPF();
3414 APFloat Val(ArgVal.getSemantics(), 1.0);
3415 APFloat::opStatus Status = Val.divide(ArgVal,
3416 APFloat::rmNearestTiesToEven);
3417 // Only do this if it was exact and therefore not dependent on the
3418 // rounding mode.
3419 if (Status == APFloat::opOK)
Sanjay Patel4b198802016-02-01 22:23:39 +00003420 return replaceInstUsesWith(CI, ConstantFP::get(II->getContext(), Val));
Matt Arsenaulta0050b02014-06-19 01:19:19 +00003421 }
3422
3423 break;
3424 }
Matt Arsenault4c7795d2017-03-24 19:04:57 +00003425 case Intrinsic::amdgcn_rsq: {
3426 Value *Src = II->getArgOperand(0);
3427
3428 // TODO: Move to ConstantFolding/InstSimplify?
3429 if (isa<UndefValue>(Src))
3430 return replaceInstUsesWith(CI, Src);
3431 break;
3432 }
Matt Arsenault2fe4fbc2016-03-30 22:28:52 +00003433 case Intrinsic::amdgcn_frexp_mant:
3434 case Intrinsic::amdgcn_frexp_exp: {
Matt Arsenault5cd4f8f2016-03-30 22:28:26 +00003435 Value *Src = II->getArgOperand(0);
3436 if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
3437 int Exp;
3438 APFloat Significand = frexp(C->getValueAPF(), Exp,
3439 APFloat::rmNearestTiesToEven);
3440
Sanjay Patel62f457b2019-05-06 15:35:02 +00003441 if (IID == Intrinsic::amdgcn_frexp_mant) {
Matt Arsenault2fe4fbc2016-03-30 22:28:52 +00003442 return replaceInstUsesWith(CI, ConstantFP::get(II->getContext(),
3443 Significand));
3444 }
3445
3446 // Match instruction special case behavior.
3447 if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
3448 Exp = 0;
3449
3450 return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Exp));
3451 }
3452
3453 if (isa<UndefValue>(Src))
3454 return replaceInstUsesWith(CI, UndefValue::get(II->getType()));
Matt Arsenault5cd4f8f2016-03-30 22:28:26 +00003455
3456 break;
3457 }
Matt Arsenault46a03822016-09-03 07:06:58 +00003458 case Intrinsic::amdgcn_class: {
3459 enum {
3460 S_NAN = 1 << 0, // Signaling NaN
3461 Q_NAN = 1 << 1, // Quiet NaN
3462 N_INFINITY = 1 << 2, // Negative infinity
3463 N_NORMAL = 1 << 3, // Negative normal
3464 N_SUBNORMAL = 1 << 4, // Negative subnormal
3465 N_ZERO = 1 << 5, // Negative zero
3466 P_ZERO = 1 << 6, // Positive zero
3467 P_SUBNORMAL = 1 << 7, // Positive subnormal
3468 P_NORMAL = 1 << 8, // Positive normal
3469 P_INFINITY = 1 << 9 // Positive infinity
3470 };
3471
3472 const uint32_t FullMask = S_NAN | Q_NAN | N_INFINITY | N_NORMAL |
3473 N_SUBNORMAL | N_ZERO | P_ZERO | P_SUBNORMAL | P_NORMAL | P_INFINITY;
3474
3475 Value *Src0 = II->getArgOperand(0);
3476 Value *Src1 = II->getArgOperand(1);
3477 const ConstantInt *CMask = dyn_cast<ConstantInt>(Src1);
3478 if (!CMask) {
3479 if (isa<UndefValue>(Src0))
3480 return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
3481
3482 if (isa<UndefValue>(Src1))
3483 return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), false));
3484 break;
3485 }
3486
3487 uint32_t Mask = CMask->getZExtValue();
3488
3489 // If all tests are made, it doesn't matter what the value is.
3490 if ((Mask & FullMask) == FullMask)
3491 return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), true));
3492
3493 if ((Mask & FullMask) == 0)
3494 return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), false));
3495
3496 if (Mask == (S_NAN | Q_NAN)) {
3497 // Equivalent of isnan. Replace with standard fcmp.
Craig Topperbb4069e2017-07-07 23:16:26 +00003498 Value *FCmp = Builder.CreateFCmpUNO(Src0, Src0);
Matt Arsenault46a03822016-09-03 07:06:58 +00003499 FCmp->takeName(II);
3500 return replaceInstUsesWith(*II, FCmp);
3501 }
3502
Matt Arsenaultd35f46c2018-08-10 18:58:49 +00003503 if (Mask == (N_ZERO | P_ZERO)) {
3504 // Equivalent of == 0.
3505 Value *FCmp = Builder.CreateFCmpOEQ(
3506 Src0, ConstantFP::get(Src0->getType(), 0.0));
3507
3508 FCmp->takeName(II);
3509 return replaceInstUsesWith(*II, FCmp);
3510 }
3511
Matt Arsenault10de2772018-08-28 18:10:02 +00003512 // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
3513 if (((Mask & S_NAN) || (Mask & Q_NAN)) && isKnownNeverNaN(Src0, &TLI)) {
3514 II->setArgOperand(1, ConstantInt::get(Src1->getType(),
3515 Mask & ~(S_NAN | Q_NAN)));
3516 return II;
3517 }
3518
Matt Arsenault46a03822016-09-03 07:06:58 +00003519 const ConstantFP *CVal = dyn_cast<ConstantFP>(Src0);
3520 if (!CVal) {
3521 if (isa<UndefValue>(Src0))
3522 return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
3523
3524 // Clamp mask to used bits
3525 if ((Mask & FullMask) != Mask) {
Craig Topperbb4069e2017-07-07 23:16:26 +00003526 CallInst *NewCall = Builder.CreateCall(II->getCalledFunction(),
Matt Arsenault46a03822016-09-03 07:06:58 +00003527 { Src0, ConstantInt::get(Src1->getType(), Mask & FullMask) }
3528 );
3529
3530 NewCall->takeName(II);
3531 return replaceInstUsesWith(*II, NewCall);
3532 }
3533
3534 break;
3535 }
3536
3537 const APFloat &Val = CVal->getValueAPF();
3538
3539 bool Result =
3540 ((Mask & S_NAN) && Val.isNaN() && Val.isSignaling()) ||
3541 ((Mask & Q_NAN) && Val.isNaN() && !Val.isSignaling()) ||
3542 ((Mask & N_INFINITY) && Val.isInfinity() && Val.isNegative()) ||
3543 ((Mask & N_NORMAL) && Val.isNormal() && Val.isNegative()) ||
3544 ((Mask & N_SUBNORMAL) && Val.isDenormal() && Val.isNegative()) ||
3545 ((Mask & N_ZERO) && Val.isZero() && Val.isNegative()) ||
3546 ((Mask & P_ZERO) && Val.isZero() && !Val.isNegative()) ||
3547 ((Mask & P_SUBNORMAL) && Val.isDenormal() && !Val.isNegative()) ||
3548 ((Mask & P_NORMAL) && Val.isNormal() && !Val.isNegative()) ||
3549 ((Mask & P_INFINITY) && Val.isInfinity() && !Val.isNegative());
3550
3551 return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), Result));
3552 }
Matt Arsenault1f17c662017-02-22 00:27:34 +00003553 case Intrinsic::amdgcn_cvt_pkrtz: {
3554 Value *Src0 = II->getArgOperand(0);
3555 Value *Src1 = II->getArgOperand(1);
3556 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
3557 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
3558 const fltSemantics &HalfSem
3559 = II->getType()->getScalarType()->getFltSemantics();
3560 bool LosesInfo;
3561 APFloat Val0 = C0->getValueAPF();
3562 APFloat Val1 = C1->getValueAPF();
3563 Val0.convert(HalfSem, APFloat::rmTowardZero, &LosesInfo);
3564 Val1.convert(HalfSem, APFloat::rmTowardZero, &LosesInfo);
3565
3566 Constant *Folded = ConstantVector::get({
3567 ConstantFP::get(II->getContext(), Val0),
3568 ConstantFP::get(II->getContext(), Val1) });
3569 return replaceInstUsesWith(*II, Folded);
3570 }
3571 }
3572
3573 if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1))
3574 return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
3575
3576 break;
3577 }
Marek Olsak13e47412018-01-31 20:18:04 +00003578 case Intrinsic::amdgcn_cvt_pknorm_i16:
3579 case Intrinsic::amdgcn_cvt_pknorm_u16:
3580 case Intrinsic::amdgcn_cvt_pk_i16:
3581 case Intrinsic::amdgcn_cvt_pk_u16: {
3582 Value *Src0 = II->getArgOperand(0);
3583 Value *Src1 = II->getArgOperand(1);
3584
3585 if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1))
3586 return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
3587
3588 break;
3589 }
Matt Arsenaultf5262252017-02-22 23:04:58 +00003590 case Intrinsic::amdgcn_ubfe:
3591 case Intrinsic::amdgcn_sbfe: {
3592 // Decompose simple cases into standard shifts.
3593 Value *Src = II->getArgOperand(0);
3594 if (isa<UndefValue>(Src))
3595 return replaceInstUsesWith(*II, Src);
3596
3597 unsigned Width;
3598 Type *Ty = II->getType();
3599 unsigned IntSize = Ty->getIntegerBitWidth();
3600
3601 ConstantInt *CWidth = dyn_cast<ConstantInt>(II->getArgOperand(2));
3602 if (CWidth) {
3603 Width = CWidth->getZExtValue();
3604 if ((Width & (IntSize - 1)) == 0)
3605 return replaceInstUsesWith(*II, ConstantInt::getNullValue(Ty));
3606
3607 if (Width >= IntSize) {
3608 // Hardware ignores high bits, so remove those.
3609 II->setArgOperand(2, ConstantInt::get(CWidth->getType(),
3610 Width & (IntSize - 1)));
3611 return II;
3612 }
3613 }
3614
3615 unsigned Offset;
3616 ConstantInt *COffset = dyn_cast<ConstantInt>(II->getArgOperand(1));
3617 if (COffset) {
3618 Offset = COffset->getZExtValue();
3619 if (Offset >= IntSize) {
3620 II->setArgOperand(1, ConstantInt::get(COffset->getType(),
3621 Offset & (IntSize - 1)));
3622 return II;
3623 }
3624 }
3625
Sanjay Patel62f457b2019-05-06 15:35:02 +00003626 bool Signed = IID == Intrinsic::amdgcn_sbfe;
Matt Arsenaultf5262252017-02-22 23:04:58 +00003627
Matt Arsenaultf5262252017-02-22 23:04:58 +00003628 if (!CWidth || !COffset)
3629 break;
3630
Tom Stellard28d66212018-11-08 17:57:57 +00003631 // The case of Width == 0 is handled above, which makes this tranformation
3632 // safe. If Width == 0, then the ashr and lshr instructions become poison
3633 // value since the shift amount would be equal to the bit size.
3634 assert(Width != 0);
3635
Matt Arsenaultf5262252017-02-22 23:04:58 +00003636 // TODO: This allows folding to undef when the hardware has specific
3637 // behavior?
3638 if (Offset + Width < IntSize) {
Craig Topperbb4069e2017-07-07 23:16:26 +00003639 Value *Shl = Builder.CreateShl(Src, IntSize - Offset - Width);
3640 Value *RightShift = Signed ? Builder.CreateAShr(Shl, IntSize - Width)
3641 : Builder.CreateLShr(Shl, IntSize - Width);
Matt Arsenaultf5262252017-02-22 23:04:58 +00003642 RightShift->takeName(II);
3643 return replaceInstUsesWith(*II, RightShift);
3644 }
3645
Craig Topperbb4069e2017-07-07 23:16:26 +00003646 Value *RightShift = Signed ? Builder.CreateAShr(Src, Offset)
3647 : Builder.CreateLShr(Src, Offset);
Matt Arsenaultf5262252017-02-22 23:04:58 +00003648
3649 RightShift->takeName(II);
3650 return replaceInstUsesWith(*II, RightShift);
3651 }
Matt Arsenaultd4bca1e2017-02-23 00:44:03 +00003652 case Intrinsic::amdgcn_exp:
3653 case Intrinsic::amdgcn_exp_compr: {
Matt Arsenaultcaf13162019-03-12 21:02:54 +00003654 ConstantInt *En = cast<ConstantInt>(II->getArgOperand(1));
Matt Arsenaultd4bca1e2017-02-23 00:44:03 +00003655 unsigned EnBits = En->getZExtValue();
3656 if (EnBits == 0xf)
3657 break; // All inputs enabled.
3658
Sanjay Patel62f457b2019-05-06 15:35:02 +00003659 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
Matt Arsenaultd4bca1e2017-02-23 00:44:03 +00003660 bool Changed = false;
3661 for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
3662 if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
3663 (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
3664 Value *Src = II->getArgOperand(I + 2);
3665 if (!isa<UndefValue>(Src)) {
3666 II->setArgOperand(I + 2, UndefValue::get(Src->getType()));
3667 Changed = true;
3668 }
3669 }
3670 }
3671
3672 if (Changed)
3673 return II;
3674
3675 break;
Matt Arsenaultcdb468c2017-02-27 23:08:49 +00003676 }
3677 case Intrinsic::amdgcn_fmed3: {
3678 // Note this does not preserve proper sNaN behavior if IEEE-mode is enabled
3679 // for the shader.
3680
3681 Value *Src0 = II->getArgOperand(0);
3682 Value *Src1 = II->getArgOperand(1);
3683 Value *Src2 = II->getArgOperand(2);
3684
Matt Arsenault24ce89b2018-07-05 17:05:36 +00003685 // Checking for NaN before canonicalization provides better fidelity when
3686 // mapping other operations onto fmed3 since the order of operands is
3687 // unchanged.
3688 CallInst *NewCall = nullptr;
3689 if (match(Src0, m_NaN()) || isa<UndefValue>(Src0)) {
3690 NewCall = Builder.CreateMinNum(Src1, Src2);
3691 } else if (match(Src1, m_NaN()) || isa<UndefValue>(Src1)) {
3692 NewCall = Builder.CreateMinNum(Src0, Src2);
3693 } else if (match(Src2, m_NaN()) || isa<UndefValue>(Src2)) {
3694 NewCall = Builder.CreateMaxNum(Src0, Src1);
3695 }
3696
3697 if (NewCall) {
3698 NewCall->copyFastMathFlags(II);
3699 NewCall->takeName(II);
3700 return replaceInstUsesWith(*II, NewCall);
3701 }
3702
Matt Arsenaultcdb468c2017-02-27 23:08:49 +00003703 bool Swap = false;
3704 // Canonicalize constants to RHS operands.
3705 //
3706 // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
3707 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
3708 std::swap(Src0, Src1);
3709 Swap = true;
3710 }
3711
3712 if (isa<Constant>(Src1) && !isa<Constant>(Src2)) {
3713 std::swap(Src1, Src2);
3714 Swap = true;
3715 }
3716
3717 if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
3718 std::swap(Src0, Src1);
3719 Swap = true;
3720 }
3721
3722 if (Swap) {
3723 II->setArgOperand(0, Src0);
3724 II->setArgOperand(1, Src1);
3725 II->setArgOperand(2, Src2);
3726 return II;
3727 }
3728
Matt Arsenaultcdb468c2017-02-27 23:08:49 +00003729 if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
3730 if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
3731 if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Src2)) {
3732 APFloat Result = fmed3AMDGCN(C0->getValueAPF(), C1->getValueAPF(),
3733 C2->getValueAPF());
3734 return replaceInstUsesWith(*II,
Craig Topperbb4069e2017-07-07 23:16:26 +00003735 ConstantFP::get(Builder.getContext(), Result));
Matt Arsenaultcdb468c2017-02-27 23:08:49 +00003736 }
3737 }
3738 }
3739
3740 break;
Matt Arsenaultd4bca1e2017-02-23 00:44:03 +00003741 }
Matt Arsenaultd81f5572017-03-13 18:14:02 +00003742 case Intrinsic::amdgcn_icmp:
3743 case Intrinsic::amdgcn_fcmp: {
Matt Arsenaultcaf13162019-03-12 21:02:54 +00003744 const ConstantInt *CC = cast<ConstantInt>(II->getArgOperand(2));
Matt Arsenaultd81f5572017-03-13 18:14:02 +00003745 // Guard against invalid arguments.
3746 int64_t CCVal = CC->getZExtValue();
Sanjay Patel62f457b2019-05-06 15:35:02 +00003747 bool IsInteger = IID == Intrinsic::amdgcn_icmp;
Matt Arsenaultd81f5572017-03-13 18:14:02 +00003748 if ((IsInteger && (CCVal < CmpInst::FIRST_ICMP_PREDICATE ||
3749 CCVal > CmpInst::LAST_ICMP_PREDICATE)) ||
3750 (!IsInteger && (CCVal < CmpInst::FIRST_FCMP_PREDICATE ||
3751 CCVal > CmpInst::LAST_FCMP_PREDICATE)))
3752 break;
3753
3754 Value *Src0 = II->getArgOperand(0);
3755 Value *Src1 = II->getArgOperand(1);
3756
3757 if (auto *CSrc0 = dyn_cast<Constant>(Src0)) {
3758 if (auto *CSrc1 = dyn_cast<Constant>(Src1)) {
3759 Constant *CCmp = ConstantExpr::getCompare(CCVal, CSrc0, CSrc1);
Nicolai Haehnle9c661852017-04-24 17:08:43 +00003760 if (CCmp->isNullValue()) {
3761 return replaceInstUsesWith(
3762 *II, ConstantExpr::getSExt(CCmp, II->getType()));
3763 }
3764
3765 // The result of V_ICMP/V_FCMP assembly instructions (which this
3766 // intrinsic exposes) is one bit per thread, masked with the EXEC
3767 // register (which contains the bitmask of live threads). So a
3768 // comparison that always returns true is the same as a read of the
3769 // EXEC register.
James Y Knight7976eb52019-02-01 20:43:25 +00003770 Function *NewF = Intrinsic::getDeclaration(
Nicolai Haehnle9c661852017-04-24 17:08:43 +00003771 II->getModule(), Intrinsic::read_register, II->getType());
3772 Metadata *MDArgs[] = {MDString::get(II->getContext(), "exec")};
3773 MDNode *MD = MDNode::get(II->getContext(), MDArgs);
3774 Value *Args[] = {MetadataAsValue::get(II->getContext(), MD)};
Craig Topperbb4069e2017-07-07 23:16:26 +00003775 CallInst *NewCall = Builder.CreateCall(NewF, Args);
Nicolai Haehnle9c661852017-04-24 17:08:43 +00003776 NewCall->addAttribute(AttributeList::FunctionIndex,
3777 Attribute::Convergent);
3778 NewCall->takeName(II);
3779 return replaceInstUsesWith(*II, NewCall);
Matt Arsenaultd81f5572017-03-13 18:14:02 +00003780 }
3781
3782 // Canonicalize constants to RHS.
3783 CmpInst::Predicate SwapPred
3784 = CmpInst::getSwappedPredicate(static_cast<CmpInst::Predicate>(CCVal));
3785 II->setArgOperand(0, Src1);
3786 II->setArgOperand(1, Src0);
3787 II->setArgOperand(2, ConstantInt::get(CC->getType(),
3788 static_cast<int>(SwapPred)));
3789 return II;
3790 }
3791
3792 if (CCVal != CmpInst::ICMP_EQ && CCVal != CmpInst::ICMP_NE)
3793 break;
3794
3795 // Canonicalize compare eq with true value to compare != 0
3796 // llvm.amdgcn.icmp(zext (i1 x), 1, eq)
3797 // -> llvm.amdgcn.icmp(zext (i1 x), 0, ne)
3798 // llvm.amdgcn.icmp(sext (i1 x), -1, eq)
3799 // -> llvm.amdgcn.icmp(sext (i1 x), 0, ne)
3800 Value *ExtSrc;
3801 if (CCVal == CmpInst::ICMP_EQ &&
3802 ((match(Src1, m_One()) && match(Src0, m_ZExt(m_Value(ExtSrc)))) ||
3803 (match(Src1, m_AllOnes()) && match(Src0, m_SExt(m_Value(ExtSrc))))) &&
3804 ExtSrc->getType()->isIntegerTy(1)) {
3805 II->setArgOperand(1, ConstantInt::getNullValue(Src1->getType()));
3806 II->setArgOperand(2, ConstantInt::get(CC->getType(), CmpInst::ICMP_NE));
3807 return II;
3808 }
3809
3810 CmpInst::Predicate SrcPred;
3811 Value *SrcLHS;
3812 Value *SrcRHS;
3813
3814 // Fold compare eq/ne with 0 from a compare result as the predicate to the
3815 // intrinsic. The typical use is a wave vote function in the library, which
3816 // will be fed from a user code condition compared with 0. Fold in the
3817 // redundant compare.
3818
3819 // llvm.amdgcn.icmp([sz]ext ([if]cmp pred a, b), 0, ne)
3820 // -> llvm.amdgcn.[if]cmp(a, b, pred)
3821 //
3822 // llvm.amdgcn.icmp([sz]ext ([if]cmp pred a, b), 0, eq)
3823 // -> llvm.amdgcn.[if]cmp(a, b, inv pred)
3824 if (match(Src1, m_Zero()) &&
3825 match(Src0,
3826 m_ZExtOrSExt(m_Cmp(SrcPred, m_Value(SrcLHS), m_Value(SrcRHS))))) {
3827 if (CCVal == CmpInst::ICMP_EQ)
3828 SrcPred = CmpInst::getInversePredicate(SrcPred);
3829
3830 Intrinsic::ID NewIID = CmpInst::isFPPredicate(SrcPred) ?
3831 Intrinsic::amdgcn_fcmp : Intrinsic::amdgcn_icmp;
3832
Matt Arsenault9a389fb2018-08-15 21:14:25 +00003833 Type *Ty = SrcLHS->getType();
3834 if (auto *CmpType = dyn_cast<IntegerType>(Ty)) {
3835 // Promote to next legal integer type.
3836 unsigned Width = CmpType->getBitWidth();
3837 unsigned NewWidth = Width;
Marek Olsak33eb4d92019-01-15 02:13:18 +00003838
3839 // Don't do anything for i1 comparisons.
3840 if (Width == 1)
3841 break;
3842
Matt Arsenault9a389fb2018-08-15 21:14:25 +00003843 if (Width <= 16)
3844 NewWidth = 16;
3845 else if (Width <= 32)
3846 NewWidth = 32;
3847 else if (Width <= 64)
3848 NewWidth = 64;
3849 else if (Width > 64)
3850 break; // Can't handle this.
3851
3852 if (Width != NewWidth) {
3853 IntegerType *CmpTy = Builder.getIntNTy(NewWidth);
3854 if (CmpInst::isSigned(SrcPred)) {
3855 SrcLHS = Builder.CreateSExt(SrcLHS, CmpTy);
3856 SrcRHS = Builder.CreateSExt(SrcRHS, CmpTy);
3857 } else {
3858 SrcLHS = Builder.CreateZExt(SrcLHS, CmpTy);
3859 SrcRHS = Builder.CreateZExt(SrcRHS, CmpTy);
3860 }
3861 }
3862 } else if (!Ty->isFloatTy() && !Ty->isDoubleTy() && !Ty->isHalfTy())
3863 break;
3864
James Y Knight7976eb52019-02-01 20:43:25 +00003865 Function *NewF =
3866 Intrinsic::getDeclaration(II->getModule(), NewIID, SrcLHS->getType());
Matt Arsenaultd81f5572017-03-13 18:14:02 +00003867 Value *Args[] = { SrcLHS, SrcRHS,
3868 ConstantInt::get(CC->getType(), SrcPred) };
Craig Topperbb4069e2017-07-07 23:16:26 +00003869 CallInst *NewCall = Builder.CreateCall(NewF, Args);
Matt Arsenaultd81f5572017-03-13 18:14:02 +00003870 NewCall->takeName(II);
3871 return replaceInstUsesWith(*II, NewCall);
3872 }
3873
3874 break;
3875 }
Marek Olsak2114fc32017-10-24 10:26:59 +00003876 case Intrinsic::amdgcn_wqm_vote: {
3877 // wqm_vote is identity when the argument is constant.
3878 if (!isa<Constant>(II->getArgOperand(0)))
3879 break;
3880
3881 return replaceInstUsesWith(*II, II->getArgOperand(0));
3882 }
Marek Olsakce76ea02017-10-24 10:27:13 +00003883 case Intrinsic::amdgcn_kill: {
3884 const ConstantInt *C = dyn_cast<ConstantInt>(II->getArgOperand(0));
3885 if (!C || !C->getZExtValue())
3886 break;
3887
3888 // amdgcn.kill(i1 1) is a no-op
3889 return eraseInstFromFunction(CI);
3890 }
Stanislav Mekhanoshin0e132dc2018-05-22 08:04:33 +00003891 case Intrinsic::amdgcn_update_dpp: {
3892 Value *Old = II->getArgOperand(0);
3893
Matt Arsenaultcaf13162019-03-12 21:02:54 +00003894 auto BC = cast<ConstantInt>(II->getArgOperand(5));
3895 auto RM = cast<ConstantInt>(II->getArgOperand(3));
3896 auto BM = cast<ConstantInt>(II->getArgOperand(4));
3897 if (BC->isZeroValue() ||
Stanislav Mekhanoshin0e132dc2018-05-22 08:04:33 +00003898 RM->getZExtValue() != 0xF ||
3899 BM->getZExtValue() != 0xF ||
3900 isa<UndefValue>(Old))
3901 break;
3902
3903 // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
3904 II->setOperand(0, UndefValue::get(Old->getType()));
3905 return II;
3906 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003907 case Intrinsic::stackrestore: {
3908 // If the save is right next to the restore, remove the restore. This can
3909 // happen when variable allocas are DCE'd.
Gabor Greif589a0b92010-06-24 12:58:35 +00003910 if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003911 if (SS->getIntrinsicID() == Intrinsic::stacksave) {
Vedant Kumarf01827f2018-06-19 23:42:17 +00003912 // Skip over debug info.
3913 if (SS->getNextNonDebugInstruction() == II) {
Sanjay Patel4b198802016-02-01 22:23:39 +00003914 return eraseInstFromFunction(CI);
Davide Italiano189c2cf2018-06-08 20:42:36 +00003915 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003916 }
3917 }
Jim Grosbach7815f562012-02-03 00:07:04 +00003918
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003919 // Scan down this block to see if there is another stack restore in the
3920 // same block without an intervening call/alloca.
Duncan P. N. Exon Smith9f8aaf22015-10-13 16:59:33 +00003921 BasicBlock::iterator BI(II);
Chandler Carruthedb12a82018-10-15 10:04:59 +00003922 Instruction *TI = II->getParent()->getTerminator();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003923 bool CannotRemove = false;
3924 for (++BI; &*BI != TI; ++BI) {
Nuno Lopes55fff832012-06-21 15:45:28 +00003925 if (isa<AllocaInst>(BI)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003926 CannotRemove = true;
3927 break;
3928 }
3929 if (CallInst *BCI = dyn_cast<CallInst>(BI)) {
Sanjay Patel62f457b2019-05-06 15:35:02 +00003930 if (auto *II2 = dyn_cast<IntrinsicInst>(BCI)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003931 // If there is a stackrestore below this one, remove this one.
Sanjay Patel62f457b2019-05-06 15:35:02 +00003932 if (II2->getIntrinsicID() == Intrinsic::stackrestore)
Sanjay Patel4b198802016-02-01 22:23:39 +00003933 return eraseInstFromFunction(CI);
Reid Kleckner892ae2e2016-02-27 00:53:54 +00003934
3935 // Bail if we cross over an intrinsic with side effects, such as
3936 // llvm.stacksave, llvm.read_register, or llvm.setjmp.
Sanjay Patel62f457b2019-05-06 15:35:02 +00003937 if (II2->mayHaveSideEffects()) {
Reid Kleckner892ae2e2016-02-27 00:53:54 +00003938 CannotRemove = true;
3939 break;
3940 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003941 } else {
3942 // If we found a non-intrinsic call, we can't remove the stack
3943 // restore.
3944 CannotRemove = true;
3945 break;
3946 }
3947 }
3948 }
Jim Grosbach7815f562012-02-03 00:07:04 +00003949
Bill Wendlingf891bf82011-07-31 06:30:59 +00003950 // If the stack restore is in a return, resume, or unwind block and if there
3951 // are no allocas or calls between the restore and the return, nuke the
3952 // restore.
Bill Wendlingd5d95b02012-02-06 21:16:41 +00003953 if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI)))
Sanjay Patel4b198802016-02-01 22:23:39 +00003954 return eraseInstFromFunction(CI);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00003955 break;
3956 }
Vitaly Bukaf0500b62016-07-28 22:50:48 +00003957 case Intrinsic::lifetime_start:
Vitaly Buka0ab23cf2016-07-28 22:59:03 +00003958 // Asan needs to poison memory to detect invalid access which is possible
3959 // even for empty lifetime range.
Evgeniy Stepanovc667c1f2017-12-09 00:21:41 +00003960 if (II->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
3961 II->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
Vitaly Buka0ab23cf2016-07-28 22:59:03 +00003962 break;
3963
Arnaud A. de Grandmaison333ef382016-05-10 09:24:49 +00003964 if (removeTriviallyEmptyRange(*II, Intrinsic::lifetime_start,
3965 Intrinsic::lifetime_end, *this))
3966 return nullptr;
Arnaud A. de Grandmaison849f3bf2015-10-01 14:54:31 +00003967 break;
Hal Finkelf5867a72014-07-25 21:45:17 +00003968 case Intrinsic::assume: {
David Majnemerfcc58112016-04-08 16:37:12 +00003969 Value *IIOperand = II->getArgOperand(0);
Sanjay Patel825a4fa2018-06-20 13:22:26 +00003970 // Remove an assume if it is followed by an identical assume.
3971 // TODO: Do we need this? Unless there are conflicting assumptions, the
3972 // computeKnownBits(IIOperand) below here eliminates redundant assumes.
3973 Instruction *Next = II->getNextNonDebugInstruction();
3974 if (match(Next, m_Intrinsic<Intrinsic::assume>(m_Specific(IIOperand))))
David Majnemerfcc58112016-04-08 16:37:12 +00003975 return eraseInstFromFunction(CI);
3976
Hal Finkelf5867a72014-07-25 21:45:17 +00003977 // Canonicalize assume(a && b) -> assume(a); assume(b);
Hal Finkel74c2f352014-09-07 12:44:26 +00003978 // Note: New assumption intrinsics created here are registered by
3979 // the InstCombineIRInserter object.
James Y Knight7976eb52019-02-01 20:43:25 +00003980 FunctionType *AssumeIntrinsicTy = II->getFunctionType();
3981 Value *AssumeIntrinsic = II->getCalledValue();
3982 Value *A, *B;
Hal Finkelf5867a72014-07-25 21:45:17 +00003983 if (match(IIOperand, m_And(m_Value(A), m_Value(B)))) {
James Y Knight7976eb52019-02-01 20:43:25 +00003984 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, A, II->getName());
3985 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, B, II->getName());
Sanjay Patel4b198802016-02-01 22:23:39 +00003986 return eraseInstFromFunction(*II);
Hal Finkelf5867a72014-07-25 21:45:17 +00003987 }
3988 // assume(!(a || b)) -> assume(!a); assume(!b);
3989 if (match(IIOperand, m_Not(m_Or(m_Value(A), m_Value(B))))) {
James Y Knight7976eb52019-02-01 20:43:25 +00003990 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
3991 Builder.CreateNot(A), II->getName());
3992 Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
3993 Builder.CreateNot(B), II->getName());
Sanjay Patel4b198802016-02-01 22:23:39 +00003994 return eraseInstFromFunction(*II);
Hal Finkelf5867a72014-07-25 21:45:17 +00003995 }
Hal Finkel04a15612014-10-04 21:27:06 +00003996
Philip Reames66c6de62014-11-11 23:33:19 +00003997 // assume( (load addr) != null ) -> add 'nonnull' metadata to load
3998 // (if assume is valid at the load)
Sanjay Patelf0d1e7732017-01-03 22:25:31 +00003999 CmpInst::Predicate Pred;
4000 Instruction *LHS;
4001 if (match(IIOperand, m_ICmp(Pred, m_Instruction(LHS), m_Zero())) &&
4002 Pred == ICmpInst::ICMP_NE && LHS->getOpcode() == Instruction::Load &&
4003 LHS->getType()->isPointerTy() &&
4004 isValidAssumeForContext(II, LHS, &DT)) {
4005 MDNode *MD = MDNode::get(II->getContext(), None);
4006 LHS->setMetadata(LLVMContext::MD_nonnull, MD);
4007 return eraseInstFromFunction(*II);
4008
Chandler Carruth24969102015-02-10 08:07:32 +00004009 // TODO: apply nonnull return attributes to calls and invokes
Philip Reames66c6de62014-11-11 23:33:19 +00004010 // TODO: apply range metadata for range check patterns?
4011 }
Sanjay Patelf0d1e7732017-01-03 22:25:31 +00004012
Hal Finkel04a15612014-10-04 21:27:06 +00004013 // If there is a dominating assume with the same condition as this one,
4014 // then this one is redundant, and should be removed.
Craig Topperb45eabc2017-04-26 16:39:58 +00004015 KnownBits Known(1);
4016 computeKnownBits(IIOperand, Known, 0, II);
Craig Topperf0aeee02017-05-05 17:36:09 +00004017 if (Known.isAllOnes())
Sanjay Patel4b198802016-02-01 22:23:39 +00004018 return eraseInstFromFunction(*II);
Hal Finkel04a15612014-10-04 21:27:06 +00004019
Hal Finkel8a9a7832017-01-11 13:24:24 +00004020 // Update the cache of affected values for this assumption (we might be
4021 // here because we just simplified the condition).
4022 AC.updateAffectedValues(II);
Hal Finkelf5867a72014-07-25 21:45:17 +00004023 break;
4024 }
Philip Reames9db26ff2014-12-29 23:27:30 +00004025 case Intrinsic::experimental_gc_relocate: {
4026 // Translate facts known about a pointer before relocating into
4027 // facts about the relocate value, while being careful to
4028 // preserve relocation semantics.
Manuel Jacob83eefa62016-01-05 04:03:00 +00004029 Value *DerivedPtr = cast<GCRelocateInst>(II)->getDerivedPtr();
Philip Reames9db26ff2014-12-29 23:27:30 +00004030
4031 // Remove the relocation if unused, note that this check is required
4032 // to prevent the cases below from looping forever.
4033 if (II->use_empty())
Sanjay Patel4b198802016-02-01 22:23:39 +00004034 return eraseInstFromFunction(*II);
Philip Reames9db26ff2014-12-29 23:27:30 +00004035
4036 // Undef is undef, even after relocation.
4037 // TODO: provide a hook for this in GCStrategy. This is clearly legal for
4038 // most practical collectors, but there was discussion in the review thread
4039 // about whether it was legal for all possible collectors.
Philip Reamesea4d8e82016-02-09 21:09:22 +00004040 if (isa<UndefValue>(DerivedPtr))
4041 // Use undef of gc_relocate's type to replace it.
4042 return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
Philip Reames9db26ff2014-12-29 23:27:30 +00004043
Philip Reamesea4d8e82016-02-09 21:09:22 +00004044 if (auto *PT = dyn_cast<PointerType>(II->getType())) {
4045 // The relocation of null will be null for most any collector.
4046 // TODO: provide a hook for this in GCStrategy. There might be some
4047 // weird collector this property does not hold for.
4048 if (isa<ConstantPointerNull>(DerivedPtr))
4049 // Use null-pointer of gc_relocate's type to replace it.
4050 return replaceInstUsesWith(*II, ConstantPointerNull::get(PT));
Simon Pilgrimc0c56e72016-04-24 17:00:34 +00004051
Philip Reamesea4d8e82016-02-09 21:09:22 +00004052 // isKnownNonNull -> nonnull attribute
Philip Reamesb8d8db32018-11-12 20:00:53 +00004053 if (!II->hasRetAttr(Attribute::NonNull) &&
4054 isKnownNonZero(DerivedPtr, DL, 0, &AC, II, &DT)) {
Reid Klecknerb5180542017-03-21 16:57:19 +00004055 II->addAttribute(AttributeList::ReturnIndex, Attribute::NonNull);
Philip Reamesb8d8db32018-11-12 20:00:53 +00004056 return II;
4057 }
Ramkumar Ramachandra8fcb4982015-02-14 19:37:54 +00004058 }
Philip Reames9db26ff2014-12-29 23:27:30 +00004059
4060 // TODO: bitcast(relocate(p)) -> relocate(bitcast(p))
4061 // Canonicalize on the type from the uses to the defs
Ramkumar Ramachandra8fcb4982015-02-14 19:37:54 +00004062
Philip Reames9db26ff2014-12-29 23:27:30 +00004063 // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...)
Philip Reamesea4d8e82016-02-09 21:09:22 +00004064 break;
Philip Reames9db26ff2014-12-29 23:27:30 +00004065 }
Artur Pilipenkoe812ca02017-01-25 14:12:12 +00004066
4067 case Intrinsic::experimental_guard: {
Philip Reames79e917d2018-05-09 22:56:32 +00004068 // Is this guard followed by another guard? We scan forward over a small
4069 // fixed window of instructions to handle common cases with conditions
4070 // computed between guards.
Sanjoy Dase0e57952017-02-01 16:34:55 +00004071 Instruction *NextInst = II->getNextNode();
Philip Reames913a7792018-05-10 00:05:29 +00004072 for (unsigned i = 0; i < GuardWideningWindow; i++) {
Philip Reames79e917d2018-05-09 22:56:32 +00004073 // Note: Using context-free form to avoid compile time blow up
4074 if (!isSafeToSpeculativelyExecute(NextInst))
4075 break;
4076 NextInst = NextInst->getNextNode();
4077 }
Sanjoy Dase0e57952017-02-01 16:34:55 +00004078 Value *NextCond = nullptr;
4079 if (match(NextInst,
4080 m_Intrinsic<Intrinsic::experimental_guard>(m_Value(NextCond)))) {
4081 Value *CurrCond = II->getArgOperand(0);
Artur Pilipenkoe812ca02017-01-25 14:12:12 +00004082
Simon Pilgrim68168d12017-03-30 12:59:53 +00004083 // Remove a guard that it is immediately preceded by an identical guard.
Sanjoy Dase0e57952017-02-01 16:34:55 +00004084 if (CurrCond == NextCond)
4085 return eraseInstFromFunction(*NextInst);
4086
4087 // Otherwise canonicalize guard(a); guard(b) -> guard(a & b).
Philip Reames79e917d2018-05-09 22:56:32 +00004088 Instruction* MoveI = II->getNextNode();
4089 while (MoveI != NextInst) {
4090 auto *Temp = MoveI;
4091 MoveI = MoveI->getNextNode();
4092 Temp->moveBefore(II);
4093 }
Craig Topperbb4069e2017-07-07 23:16:26 +00004094 II->setArgOperand(0, Builder.CreateAnd(CurrCond, NextCond));
Sanjoy Dase0e57952017-02-01 16:34:55 +00004095 return eraseInstFromFunction(*NextInst);
4096 }
Artur Pilipenkoe812ca02017-01-25 14:12:12 +00004097 break;
4098 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004099 }
Craig Topperc1892ec2019-01-31 17:23:29 +00004100 return visitCallBase(*II);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004101}
4102
Davide Italianoaec46172017-01-31 18:09:05 +00004103// Fence instruction simplification
4104Instruction *InstCombiner::visitFenceInst(FenceInst &FI) {
4105 // Remove identical consecutive fences.
Vedant Kumarf01827f2018-06-19 23:42:17 +00004106 Instruction *Next = FI.getNextNonDebugInstruction();
Tim Northover9b800602018-06-06 12:46:02 +00004107 if (auto *NFI = dyn_cast<FenceInst>(Next))
Davide Italianoaec46172017-01-31 18:09:05 +00004108 if (FI.isIdenticalTo(NFI))
4109 return eraseInstFromFunction(FI);
4110 return nullptr;
4111}
4112
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004113// InvokeInst simplification
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004114Instruction *InstCombiner::visitInvokeInst(InvokeInst &II) {
Craig Topperc1892ec2019-01-31 17:23:29 +00004115 return visitCallBase(II);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004116}
4117
Craig Topper784929d2019-02-08 20:48:56 +00004118// CallBrInst simplification
4119Instruction *InstCombiner::visitCallBrInst(CallBrInst &CBI) {
4120 return visitCallBase(CBI);
4121}
4122
Sanjay Patelcd4377c2016-01-20 22:24:38 +00004123/// If this cast does not affect the value passed through the varargs area, we
4124/// can eliminate the use of the cast.
Craig Topperc1892ec2019-01-31 17:23:29 +00004125static bool isSafeToEliminateVarargsCast(const CallBase &Call,
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004126 const DataLayout &DL,
4127 const CastInst *const CI,
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004128 const int ix) {
4129 if (!CI->isLosslessCast())
4130 return false;
4131
Philip Reames1a1bdb22014-12-02 18:50:36 +00004132 // If this is a GC intrinsic, avoid munging types. We need types for
4133 // statepoint reconstruction in SelectionDAG.
4134 // TODO: This is probably something which should be expanded to all
4135 // intrinsics since the entire point of intrinsics is that
4136 // they are understandable by the optimizer.
Craig Topperc1892ec2019-01-31 17:23:29 +00004137 if (isStatepoint(&Call) || isGCRelocate(&Call) || isGCResult(&Call))
Philip Reames1a1bdb22014-12-02 18:50:36 +00004138 return false;
4139
Reid Kleckner26af2ca2014-01-28 02:38:36 +00004140 // The size of ByVal or InAlloca arguments is derived from the type, so we
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004141 // can't change to a type with a different size. If the size were
4142 // passed explicitly we could avoid this check.
Craig Topperc1892ec2019-01-31 17:23:29 +00004143 if (!Call.isByValOrInAllocaArgument(ix))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004144 return true;
4145
Jim Grosbach7815f562012-02-03 00:07:04 +00004146 Type* SrcTy =
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004147 cast<PointerType>(CI->getOperand(0)->getType())->getElementType();
Chris Lattner229907c2011-07-18 04:54:35 +00004148 Type* DstTy = cast<PointerType>(CI->getType())->getElementType();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004149 if (!SrcTy->isSized() || !DstTy->isSized())
4150 return false;
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004151 if (DL.getTypeAllocSize(SrcTy) != DL.getTypeAllocSize(DstTy))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004152 return false;
4153 return true;
4154}
4155
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004156Instruction *InstCombiner::tryOptimizeCall(CallInst *CI) {
Craig Topperf40110f2014-04-25 05:29:35 +00004157 if (!CI->getCalledFunction()) return nullptr;
Eric Christophera7fb58f2010-03-06 10:50:38 +00004158
Chandler Carruthba4c5172015-01-21 11:23:40 +00004159 auto InstCombineRAUW = [this](Instruction *From, Value *With) {
Sanjay Patel4b198802016-02-01 22:23:39 +00004160 replaceInstUsesWith(*From, With);
Chandler Carruthba4c5172015-01-21 11:23:40 +00004161 };
Amara Emerson54f60252018-10-11 14:51:11 +00004162 auto InstCombineErase = [this](Instruction *I) {
4163 eraseInstFromFunction(*I);
4164 };
Hiroshi Yamauchi09e539f2019-04-15 16:49:00 +00004165 LibCallSimplifier Simplifier(DL, &TLI, ORE, BFI, PSI, InstCombineRAUW,
Amara Emerson54f60252018-10-11 14:51:11 +00004166 InstCombineErase);
Chandler Carruthba4c5172015-01-21 11:23:40 +00004167 if (Value *With = Simplifier.optimizeCall(CI)) {
Meador Ingee3f2b262012-11-30 04:05:06 +00004168 ++NumSimplified;
Sanjay Patel4b198802016-02-01 22:23:39 +00004169 return CI->use_empty() ? CI : replaceInstUsesWith(*CI, With);
Meador Ingee3f2b262012-11-30 04:05:06 +00004170 }
Meador Ingedf796f82012-10-13 16:45:24 +00004171
Craig Topperf40110f2014-04-25 05:29:35 +00004172 return nullptr;
Eric Christophera7fb58f2010-03-06 10:50:38 +00004173}
4174
Sanjay Patel6038d3e2016-01-29 23:27:03 +00004175static IntrinsicInst *findInitTrampolineFromAlloca(Value *TrampMem) {
Duncan Sandsa0984362011-09-06 13:37:06 +00004176 // Strip off at most one level of pointer casts, looking for an alloca. This
4177 // is good enough in practice and simpler than handling any number of casts.
4178 Value *Underlying = TrampMem->stripPointerCasts();
4179 if (Underlying != TrampMem &&
Chandler Carruthcdf47882014-03-09 03:16:01 +00004180 (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem))
Craig Topperf40110f2014-04-25 05:29:35 +00004181 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004182 if (!isa<AllocaInst>(Underlying))
Craig Topperf40110f2014-04-25 05:29:35 +00004183 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004184
Craig Topperf40110f2014-04-25 05:29:35 +00004185 IntrinsicInst *InitTrampoline = nullptr;
Chandler Carruthcdf47882014-03-09 03:16:01 +00004186 for (User *U : TrampMem->users()) {
4187 IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
Duncan Sandsa0984362011-09-06 13:37:06 +00004188 if (!II)
Craig Topperf40110f2014-04-25 05:29:35 +00004189 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004190 if (II->getIntrinsicID() == Intrinsic::init_trampoline) {
4191 if (InitTrampoline)
4192 // More than one init_trampoline writes to this value. Give up.
Craig Topperf40110f2014-04-25 05:29:35 +00004193 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004194 InitTrampoline = II;
4195 continue;
4196 }
4197 if (II->getIntrinsicID() == Intrinsic::adjust_trampoline)
4198 // Allow any number of calls to adjust.trampoline.
4199 continue;
Craig Topperf40110f2014-04-25 05:29:35 +00004200 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004201 }
4202
4203 // No call to init.trampoline found.
4204 if (!InitTrampoline)
Craig Topperf40110f2014-04-25 05:29:35 +00004205 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004206
4207 // Check that the alloca is being used in the expected way.
4208 if (InitTrampoline->getOperand(0) != TrampMem)
Craig Topperf40110f2014-04-25 05:29:35 +00004209 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004210
4211 return InitTrampoline;
4212}
4213
Sanjay Patel6038d3e2016-01-29 23:27:03 +00004214static IntrinsicInst *findInitTrampolineFromBB(IntrinsicInst *AdjustTramp,
Duncan Sandsa0984362011-09-06 13:37:06 +00004215 Value *TrampMem) {
4216 // Visit all the previous instructions in the basic block, and try to find a
4217 // init.trampoline which has a direct path to the adjust.trampoline.
Duncan P. N. Exon Smith9f8aaf22015-10-13 16:59:33 +00004218 for (BasicBlock::iterator I = AdjustTramp->getIterator(),
4219 E = AdjustTramp->getParent()->begin();
4220 I != E;) {
4221 Instruction *Inst = &*--I;
Duncan Sandsa0984362011-09-06 13:37:06 +00004222 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
4223 if (II->getIntrinsicID() == Intrinsic::init_trampoline &&
4224 II->getOperand(0) == TrampMem)
4225 return II;
4226 if (Inst->mayWriteToMemory())
Craig Topperf40110f2014-04-25 05:29:35 +00004227 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004228 }
Craig Topperf40110f2014-04-25 05:29:35 +00004229 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004230}
4231
4232// Given a call to llvm.adjust.trampoline, find and return the corresponding
4233// call to llvm.init.trampoline if the call to the trampoline can be optimized
4234// to a direct call to a function. Otherwise return NULL.
Sanjay Patel6038d3e2016-01-29 23:27:03 +00004235static IntrinsicInst *findInitTrampoline(Value *Callee) {
Duncan Sandsa0984362011-09-06 13:37:06 +00004236 Callee = Callee->stripPointerCasts();
4237 IntrinsicInst *AdjustTramp = dyn_cast<IntrinsicInst>(Callee);
4238 if (!AdjustTramp ||
4239 AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline)
Craig Topperf40110f2014-04-25 05:29:35 +00004240 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004241
4242 Value *TrampMem = AdjustTramp->getOperand(0);
4243
Sanjay Patel6038d3e2016-01-29 23:27:03 +00004244 if (IntrinsicInst *IT = findInitTrampolineFromAlloca(TrampMem))
Duncan Sandsa0984362011-09-06 13:37:06 +00004245 return IT;
Sanjay Patel6038d3e2016-01-29 23:27:03 +00004246 if (IntrinsicInst *IT = findInitTrampolineFromBB(AdjustTramp, TrampMem))
Duncan Sandsa0984362011-09-06 13:37:06 +00004247 return IT;
Craig Topperf40110f2014-04-25 05:29:35 +00004248 return nullptr;
Duncan Sandsa0984362011-09-06 13:37:06 +00004249}
4250
Craig Topper784929d2019-02-08 20:48:56 +00004251/// Improvements for call, callbr and invoke instructions.
Craig Topperc1892ec2019-01-31 17:23:29 +00004252Instruction *InstCombiner::visitCallBase(CallBase &Call) {
4253 if (isAllocLikeFn(&Call, &TLI))
4254 return visitAllocSite(Call);
Nuno Lopesdc6085e2012-06-21 21:25:05 +00004255
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004256 bool Changed = false;
4257
Philip Reamesc25df112015-06-16 20:24:25 +00004258 // Mark any parameters that are known to be non-null with the nonnull
4259 // attribute. This is helpful for inlining calls to functions with null
4260 // checks on their arguments.
Reid Kleckner5fbdd172017-05-31 19:23:09 +00004261 SmallVector<unsigned, 4> ArgNos;
Philip Reamesc25df112015-06-16 20:24:25 +00004262 unsigned ArgNo = 0;
Akira Hatanaka237916b2015-12-02 06:58:49 +00004263
Craig Topperc1892ec2019-01-31 17:23:29 +00004264 for (Value *V : Call.args()) {
Sanjay Patelf9f5d3c2016-01-29 23:14:58 +00004265 if (V->getType()->isPointerTy() &&
Craig Topperc1892ec2019-01-31 17:23:29 +00004266 !Call.paramHasAttr(ArgNo, Attribute::NonNull) &&
4267 isKnownNonZero(V, DL, 0, &AC, &Call, &DT))
Reid Kleckner5fbdd172017-05-31 19:23:09 +00004268 ArgNos.push_back(ArgNo);
Philip Reamesc25df112015-06-16 20:24:25 +00004269 ArgNo++;
4270 }
Akira Hatanaka237916b2015-12-02 06:58:49 +00004271
Craig Topperc1892ec2019-01-31 17:23:29 +00004272 assert(ArgNo == Call.arg_size() && "sanity check");
Philip Reamesc25df112015-06-16 20:24:25 +00004273
Reid Kleckner5fbdd172017-05-31 19:23:09 +00004274 if (!ArgNos.empty()) {
Craig Topperc1892ec2019-01-31 17:23:29 +00004275 AttributeList AS = Call.getAttributes();
4276 LLVMContext &Ctx = Call.getContext();
Reid Kleckner5fbdd172017-05-31 19:23:09 +00004277 AS = AS.addParamAttribute(Ctx, ArgNos,
4278 Attribute::get(Ctx, Attribute::NonNull));
Craig Topperc1892ec2019-01-31 17:23:29 +00004279 Call.setAttributes(AS);
Akira Hatanaka237916b2015-12-02 06:58:49 +00004280 Changed = true;
4281 }
4282
Chris Lattner73989652010-12-20 08:25:06 +00004283 // If the callee is a pointer to a function, attempt to move any casts to the
Craig Topper784929d2019-02-08 20:48:56 +00004284 // arguments of the call/callbr/invoke.
Craig Topperc1892ec2019-01-31 17:23:29 +00004285 Value *Callee = Call.getCalledValue();
4286 if (!isa<Function>(Callee) && transformConstExprCastCall(Call))
Craig Topperf40110f2014-04-25 05:29:35 +00004287 return nullptr;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004288
Justin Lebar9d943972016-03-14 20:18:54 +00004289 if (Function *CalleeF = dyn_cast<Function>(Callee)) {
4290 // Remove the convergent attr on calls when the callee is not convergent.
Craig Topperc1892ec2019-01-31 17:23:29 +00004291 if (Call.isConvergent() && !CalleeF->isConvergent() &&
Matt Arsenault802ebcb2016-06-20 19:04:44 +00004292 !CalleeF->isIntrinsic()) {
Craig Topperc1892ec2019-01-31 17:23:29 +00004293 LLVM_DEBUG(dbgs() << "Removing convergent attr from instr " << Call
4294 << "\n");
4295 Call.setNotConvergent();
4296 return &Call;
Justin Lebar9d943972016-03-14 20:18:54 +00004297 }
4298
Chris Lattner846a52e2010-02-01 18:11:34 +00004299 // If the call and callee calling conventions don't match, this call must
4300 // be unreachable, as the call is undefined.
Craig Topperc1892ec2019-01-31 17:23:29 +00004301 if (CalleeF->getCallingConv() != Call.getCallingConv() &&
Chris Lattner846a52e2010-02-01 18:11:34 +00004302 // Only do this for calls to a function with a body. A prototype may
4303 // not actually end up matching the implementation's calling conv for a
4304 // variety of reasons (e.g. it may be written in assembly).
4305 !CalleeF->isDeclaration()) {
Craig Topperc1892ec2019-01-31 17:23:29 +00004306 Instruction *OldCall = &Call;
Philip Reames88679712019-04-17 17:37:58 +00004307 CreateNonTerminatorUnreachable(OldCall);
Chad Rosiere28ae302012-12-13 00:18:46 +00004308 // If OldCall does not return void then replaceAllUsesWith undef.
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004309 // This allows ValueHandlers and custom metadata to adjust itself.
4310 if (!OldCall->getType()->isVoidTy())
Sanjay Patel4b198802016-02-01 22:23:39 +00004311 replaceInstUsesWith(*OldCall, UndefValue::get(OldCall->getType()));
Chris Lattner2cecedf2010-02-01 18:04:58 +00004312 if (isa<CallInst>(OldCall))
Sanjay Patel4b198802016-02-01 22:23:39 +00004313 return eraseInstFromFunction(*OldCall);
Jim Grosbach7815f562012-02-03 00:07:04 +00004314
Craig Topper784929d2019-02-08 20:48:56 +00004315 // We cannot remove an invoke or a callbr, because it would change thexi
4316 // CFG, just change the callee to a null pointer.
4317 cast<CallBase>(OldCall)->setCalledFunction(
James Y Knight291f7912019-02-01 20:44:54 +00004318 CalleeF->getFunctionType(),
4319 Constant::getNullValue(CalleeF->getType()));
Craig Topperf40110f2014-04-25 05:29:35 +00004320 return nullptr;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004321 }
Justin Lebar9d943972016-03-14 20:18:54 +00004322 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004323
Manoj Gupta77eeac32018-07-09 22:27:23 +00004324 if ((isa<ConstantPointerNull>(Callee) &&
Craig Topperc1892ec2019-01-31 17:23:29 +00004325 !NullPointerIsDefined(Call.getFunction())) ||
Manoj Gupta77eeac32018-07-09 22:27:23 +00004326 isa<UndefValue>(Callee)) {
Craig Topperc1892ec2019-01-31 17:23:29 +00004327 // If Call does not return void then replaceAllUsesWith undef.
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004328 // This allows ValueHandlers and custom metadata to adjust itself.
Craig Topperc1892ec2019-01-31 17:23:29 +00004329 if (!Call.getType()->isVoidTy())
4330 replaceInstUsesWith(Call, UndefValue::get(Call.getType()));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004331
Craig Topper784929d2019-02-08 20:48:56 +00004332 if (Call.isTerminator()) {
4333 // Can't remove an invoke or callbr because we cannot change the CFG.
Craig Topperf40110f2014-04-25 05:29:35 +00004334 return nullptr;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004335 }
Nuno Lopes771e7bd2012-06-21 23:52:14 +00004336
Philip Reames88679712019-04-17 17:37:58 +00004337 // This instruction is not reachable, just remove it.
4338 CreateNonTerminatorUnreachable(&Call);
Craig Topperc1892ec2019-01-31 17:23:29 +00004339 return eraseInstFromFunction(Call);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004340 }
4341
Sanjay Patel6038d3e2016-01-29 23:27:03 +00004342 if (IntrinsicInst *II = findInitTrampoline(Callee))
Craig Topperc1892ec2019-01-31 17:23:29 +00004343 return transformCallThroughTrampoline(Call, *II);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004344
Chris Lattner229907c2011-07-18 04:54:35 +00004345 PointerType *PTy = cast<PointerType>(Callee->getType());
4346 FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004347 if (FTy->isVarArg()) {
Eli Friedman7534b4682011-11-29 01:18:23 +00004348 int ix = FTy->getNumParams();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004349 // See if we can optimize any arguments passed through the varargs area of
4350 // the call.
Craig Topperc1892ec2019-01-31 17:23:29 +00004351 for (auto I = Call.arg_begin() + FTy->getNumParams(), E = Call.arg_end();
4352 I != E; ++I, ++ix) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004353 CastInst *CI = dyn_cast<CastInst>(*I);
Craig Topperc1892ec2019-01-31 17:23:29 +00004354 if (CI && isSafeToEliminateVarargsCast(Call, DL, CI, ix)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004355 *I = CI->getOperand(0);
4356 Changed = true;
4357 }
4358 }
4359 }
4360
Craig Topperc1892ec2019-01-31 17:23:29 +00004361 if (isa<InlineAsm>(Callee) && !Call.doesNotThrow()) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004362 // Inline asm calls cannot throw - mark them 'nounwind'.
Craig Topperc1892ec2019-01-31 17:23:29 +00004363 Call.setDoesNotThrow();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004364 Changed = true;
4365 }
4366
Micah Villmowcdfe20b2012-10-08 16:38:25 +00004367 // Try to optimize the call if possible, we require DataLayout for most of
Eric Christophera7fb58f2010-03-06 10:50:38 +00004368 // this. None of these calls are seen as possibly dead so go ahead and
4369 // delete the instruction now.
Craig Topperc1892ec2019-01-31 17:23:29 +00004370 if (CallInst *CI = dyn_cast<CallInst>(&Call)) {
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004371 Instruction *I = tryOptimizeCall(CI);
Eric Christopher1810d772010-03-06 10:59:25 +00004372 // If we changed something return the result, etc. Otherwise let
4373 // the fallthrough check.
Sanjay Patel4b198802016-02-01 22:23:39 +00004374 if (I) return eraseInstFromFunction(*I);
Eric Christophera7fb58f2010-03-06 10:50:38 +00004375 }
4376
Craig Topperc1892ec2019-01-31 17:23:29 +00004377 return Changed ? &Call : nullptr;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004378}
4379
Sanjay Patelcd4377c2016-01-20 22:24:38 +00004380/// If the callee is a constexpr cast of a function, attempt to move the cast to
Craig Topper784929d2019-02-08 20:48:56 +00004381/// the arguments of the call/callbr/invoke.
Craig Topperc1892ec2019-01-31 17:23:29 +00004382bool InstCombiner::transformConstExprCastCall(CallBase &Call) {
4383 auto *Callee = dyn_cast<Function>(Call.getCalledValue()->stripPointerCasts());
Craig Topperf40110f2014-04-25 05:29:35 +00004384 if (!Callee)
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004385 return false;
Sanjay Patel38ae83d2016-08-11 15:23:56 +00004386
Reid Kleckner298ffc62018-04-02 22:49:44 +00004387 // If this is a call to a thunk function, don't remove the cast. Thunks are
4388 // used to transparently forward all incoming parameters and outgoing return
4389 // values, so it's important to leave the cast in place.
David Majnemer4c0a6e92015-01-21 22:32:04 +00004390 if (Callee->hasFnAttribute("thunk"))
4391 return false;
Sanjay Patel38ae83d2016-08-11 15:23:56 +00004392
Reid Kleckner298ffc62018-04-02 22:49:44 +00004393 // If this is a musttail call, the callee's prototype must match the caller's
4394 // prototype with the exception of pointee types. The code below doesn't
4395 // implement that, so we can't do this transform.
4396 // TODO: Do the transform if it only requires adding pointer casts.
Craig Topperc1892ec2019-01-31 17:23:29 +00004397 if (Call.isMustTailCall())
Reid Kleckner298ffc62018-04-02 22:49:44 +00004398 return false;
4399
Craig Topperc1892ec2019-01-31 17:23:29 +00004400 Instruction *Caller = &Call;
4401 const AttributeList &CallerPAL = Call.getAttributes();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004402
4403 // Okay, this is a cast from a function to a different type. Unless doing so
4404 // would cause a type conversion of one of our arguments, change this call to
4405 // be a direct call with arguments casted to the appropriate types.
Chris Lattner229907c2011-07-18 04:54:35 +00004406 FunctionType *FT = Callee->getFunctionType();
4407 Type *OldRetTy = Caller->getType();
4408 Type *NewRetTy = FT->getReturnType();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004409
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004410 // Check to see if we are changing the return type...
4411 if (OldRetTy != NewRetTy) {
Nick Lewyckya6a17d72014-01-18 22:47:12 +00004412
4413 if (NewRetTy->isStructTy())
4414 return false; // TODO: Handle multiple return values.
4415
David Majnemer9b6b8222015-01-06 08:41:31 +00004416 if (!CastInst::isBitOrNoopPointerCastable(NewRetTy, OldRetTy, DL)) {
Matt Arsenaulte6952f22013-09-17 21:10:14 +00004417 if (Callee->isDeclaration())
4418 return false; // Cannot transform this return value.
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004419
Matt Arsenaulte6952f22013-09-17 21:10:14 +00004420 if (!Caller->use_empty() &&
4421 // void -> non-void is handled specially
4422 !NewRetTy->isVoidTy())
Frederic Rissc1892e22014-10-23 04:08:42 +00004423 return false; // Cannot transform this return value.
Matt Arsenaulte6952f22013-09-17 21:10:14 +00004424 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004425
4426 if (!CallerPAL.isEmpty() && !Caller->use_empty()) {
Reid Klecknerb5180542017-03-21 16:57:19 +00004427 AttrBuilder RAttrs(CallerPAL, AttributeList::ReturnIndex);
Pete Cooper2777d8872015-05-06 23:19:56 +00004428 if (RAttrs.overlaps(AttributeFuncs::typeIncompatible(NewRetTy)))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004429 return false; // Attribute not compatible with transformed value.
4430 }
4431
Craig Topper784929d2019-02-08 20:48:56 +00004432 // If the callbase is an invoke/callbr instruction, and the return value is
4433 // used by a PHI node in a successor, we cannot change the return type of
4434 // the call because there is no place to put the cast instruction (without
4435 // breaking the critical edge). Bail out in this case.
4436 if (!Caller->use_empty()) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004437 if (InvokeInst *II = dyn_cast<InvokeInst>(Caller))
Chandler Carruthcdf47882014-03-09 03:16:01 +00004438 for (User *U : II->users())
4439 if (PHINode *PN = dyn_cast<PHINode>(U))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004440 if (PN->getParent() == II->getNormalDest() ||
4441 PN->getParent() == II->getUnwindDest())
4442 return false;
Craig Topper784929d2019-02-08 20:48:56 +00004443 // FIXME: Be conservative for callbr to avoid a quadratic search.
Craig Toppera97857b2019-02-10 02:21:29 +00004444 if (isa<CallBrInst>(Caller))
Craig Topper784929d2019-02-08 20:48:56 +00004445 return false;
4446 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004447 }
4448
Craig Topperc1892ec2019-01-31 17:23:29 +00004449 unsigned NumActualArgs = Call.arg_size();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004450 unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);
4451
David Majnemer9b6b8222015-01-06 08:41:31 +00004452 // Prevent us turning:
4453 // declare void @takes_i32_inalloca(i32* inalloca)
4454 // call void bitcast (void (i32*)* @takes_i32_inalloca to void (i32)*)(i32 0)
4455 //
4456 // into:
4457 // call void @takes_i32_inalloca(i32* null)
David Majnemerd61a6fd2015-03-11 18:03:05 +00004458 //
4459 // Similarly, avoid folding away bitcasts of byval calls.
4460 if (Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) ||
4461 Callee->getAttributes().hasAttrSomewhere(Attribute::ByVal))
David Majnemer9b6b8222015-01-06 08:41:31 +00004462 return false;
4463
Craig Topperc1892ec2019-01-31 17:23:29 +00004464 auto AI = Call.arg_begin();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004465 for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {
Chris Lattner229907c2011-07-18 04:54:35 +00004466 Type *ParamTy = FT->getParamType(i);
4467 Type *ActTy = (*AI)->getType();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004468
David Majnemer9b6b8222015-01-06 08:41:31 +00004469 if (!CastInst::isBitOrNoopPointerCastable(ActTy, ParamTy, DL))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004470 return false; // Cannot transform this parameter value.
4471
Reid Klecknerf021fab2017-04-13 23:12:13 +00004472 if (AttrBuilder(CallerPAL.getParamAttributes(i))
4473 .overlaps(AttributeFuncs::typeIncompatible(ParamTy)))
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004474 return false; // Attribute not compatible with transformed value.
Jim Grosbach7815f562012-02-03 00:07:04 +00004475
Craig Topperc1892ec2019-01-31 17:23:29 +00004476 if (Call.isInAllocaArgument(i))
Reid Kleckner26af2ca2014-01-28 02:38:36 +00004477 return false; // Cannot transform to and from inalloca.
4478
Chris Lattner27ca8eb2010-12-20 08:36:38 +00004479 // If the parameter is passed as a byval argument, then we have to have a
4480 // sized type and the sized type has to have the same size as the old type.
Reid Klecknerf021fab2017-04-13 23:12:13 +00004481 if (ParamTy != ActTy && CallerPAL.hasParamAttribute(i, Attribute::ByVal)) {
Chris Lattner229907c2011-07-18 04:54:35 +00004482 PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy);
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004483 if (!ParamPTy || !ParamPTy->getElementType()->isSized())
Chris Lattner27ca8eb2010-12-20 08:36:38 +00004484 return false;
Jim Grosbach7815f562012-02-03 00:07:04 +00004485
Matt Arsenaultfa252722013-09-27 22:18:51 +00004486 Type *CurElTy = ActTy->getPointerElementType();
Mehdi Aminia28d91d2015-03-10 02:37:25 +00004487 if (DL.getTypeAllocSize(CurElTy) !=
4488 DL.getTypeAllocSize(ParamPTy->getElementType()))
Chris Lattner27ca8eb2010-12-20 08:36:38 +00004489 return false;
4490 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004491 }
4492
Chris Lattneradf38b32011-02-24 05:10:56 +00004493 if (Callee->isDeclaration()) {
4494 // Do not delete arguments unless we have a function body.
4495 if (FT->getNumParams() < NumActualArgs && !FT->isVarArg())
4496 return false;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004497
Chris Lattneradf38b32011-02-24 05:10:56 +00004498 // If the callee is just a declaration, don't change the varargsness of the
4499 // call. We don't want to introduce a varargs call where one doesn't
4500 // already exist.
Craig Topperc1892ec2019-01-31 17:23:29 +00004501 PointerType *APTy = cast<PointerType>(Call.getCalledValue()->getType());
Chris Lattneradf38b32011-02-24 05:10:56 +00004502 if (FT->isVarArg()!=cast<FunctionType>(APTy->getElementType())->isVarArg())
4503 return false;
Jim Grosbache84ae7b2012-02-03 00:00:55 +00004504
4505 // If both the callee and the cast type are varargs, we still have to make
4506 // sure the number of fixed parameters are the same or we have the same
4507 // ABI issues as if we introduce a varargs call.
Jim Grosbach1df8cdc2012-02-03 00:26:07 +00004508 if (FT->isVarArg() &&
4509 cast<FunctionType>(APTy->getElementType())->isVarArg() &&
4510 FT->getNumParams() !=
Jim Grosbache84ae7b2012-02-03 00:00:55 +00004511 cast<FunctionType>(APTy->getElementType())->getNumParams())
4512 return false;
Chris Lattneradf38b32011-02-24 05:10:56 +00004513 }
Jim Grosbach7815f562012-02-03 00:07:04 +00004514
Jim Grosbach0ab54182012-02-03 00:00:50 +00004515 if (FT->getNumParams() < NumActualArgs && FT->isVarArg() &&
Reid Kleckneraa0cec72017-04-19 23:17:47 +00004516 !CallerPAL.isEmpty()) {
Jim Grosbach0ab54182012-02-03 00:00:50 +00004517 // In this case we have more arguments than the new function type, but we
4518 // won't be dropping them. Check that these extra arguments have attributes
4519 // that are compatible with being a vararg call argument.
Reid Kleckneraa0cec72017-04-19 23:17:47 +00004520 unsigned SRetIdx;
4521 if (CallerPAL.hasAttrSomewhere(Attribute::StructRet, &SRetIdx) &&
4522 SRetIdx > FT->getNumParams())
4523 return false;
4524 }
Jim Grosbach7815f562012-02-03 00:07:04 +00004525
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004526 // Okay, we decided that this is a safe thing to do: go ahead and start
Chris Lattneradf38b32011-02-24 05:10:56 +00004527 // inserting cast instructions as necessary.
Reid Klecknerc3fae792017-04-13 18:11:03 +00004528 SmallVector<Value *, 8> Args;
4529 SmallVector<AttributeSet, 8> ArgAttrs;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004530 Args.reserve(NumActualArgs);
Reid Klecknerc3fae792017-04-13 18:11:03 +00004531 ArgAttrs.reserve(NumActualArgs);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004532
4533 // Get any return attributes.
Reid Klecknerb5180542017-03-21 16:57:19 +00004534 AttrBuilder RAttrs(CallerPAL, AttributeList::ReturnIndex);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004535
4536 // If the return value is not being used, the type may not be compatible
4537 // with the existing attributes. Wipe out any problematic attributes.
Pete Cooper2777d8872015-05-06 23:19:56 +00004538 RAttrs.remove(AttributeFuncs::typeIncompatible(NewRetTy));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004539
Craig Topperc1892ec2019-01-31 17:23:29 +00004540 AI = Call.arg_begin();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004541 for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
Chris Lattner229907c2011-07-18 04:54:35 +00004542 Type *ParamTy = FT->getParamType(i);
Matt Arsenaultcacbb232013-07-30 20:45:05 +00004543
Reid Klecknerc3fae792017-04-13 18:11:03 +00004544 Value *NewArg = *AI;
4545 if ((*AI)->getType() != ParamTy)
Craig Topperbb4069e2017-07-07 23:16:26 +00004546 NewArg = Builder.CreateBitOrPointerCast(*AI, ParamTy);
Reid Klecknerc3fae792017-04-13 18:11:03 +00004547 Args.push_back(NewArg);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004548
4549 // Add any parameter attributes.
Reid Klecknerf021fab2017-04-13 23:12:13 +00004550 ArgAttrs.push_back(CallerPAL.getParamAttributes(i));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004551 }
4552
4553 // If the function takes more arguments than the call was taking, add them
4554 // now.
Reid Klecknerc3fae792017-04-13 18:11:03 +00004555 for (unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004556 Args.push_back(Constant::getNullValue(FT->getParamType(i)));
Reid Klecknerc3fae792017-04-13 18:11:03 +00004557 ArgAttrs.push_back(AttributeSet());
4558 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004559
4560 // If we are removing arguments to the function, emit an obnoxious warning.
4561 if (FT->getNumParams() < NumActualArgs) {
Nick Lewycky90053a12012-12-26 22:00:35 +00004562 // TODO: if (!FT->isVarArg()) this call may be unreachable. PR14722
4563 if (FT->isVarArg()) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004564 // Add all of the arguments in their promoted form to the arg list.
4565 for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {
Chris Lattner229907c2011-07-18 04:54:35 +00004566 Type *PTy = getPromotedType((*AI)->getType());
Reid Klecknerc3fae792017-04-13 18:11:03 +00004567 Value *NewArg = *AI;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004568 if (PTy != (*AI)->getType()) {
4569 // Must promote to pass through va_arg area!
4570 Instruction::CastOps opcode =
4571 CastInst::getCastOpcode(*AI, false, PTy, false);
Craig Topperbb4069e2017-07-07 23:16:26 +00004572 NewArg = Builder.CreateCast(opcode, *AI, PTy);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004573 }
Reid Klecknerc3fae792017-04-13 18:11:03 +00004574 Args.push_back(NewArg);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004575
4576 // Add any parameter attributes.
Reid Klecknerf021fab2017-04-13 23:12:13 +00004577 ArgAttrs.push_back(CallerPAL.getParamAttributes(i));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004578 }
4579 }
4580 }
4581
Reid Klecknerc2cb5602017-04-12 00:38:00 +00004582 AttributeSet FnAttrs = CallerPAL.getFnAttributes();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004583
4584 if (NewRetTy->isVoidTy())
4585 Caller->setName(""); // Void type should not have a name.
4586
Reid Klecknerc3fae792017-04-13 18:11:03 +00004587 assert((ArgAttrs.size() == FT->getNumParams() || FT->isVarArg()) &&
4588 "missing argument attributes");
4589 LLVMContext &Ctx = Callee->getContext();
4590 AttributeList NewCallerPAL = AttributeList::get(
4591 Ctx, FnAttrs, AttributeSet::get(Ctx, RAttrs), ArgAttrs);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004592
Sanjoy Das76293462015-11-25 00:42:19 +00004593 SmallVector<OperandBundleDef, 1> OpBundles;
Craig Topperc1892ec2019-01-31 17:23:29 +00004594 Call.getOperandBundlesAsDefs(OpBundles);
Sanjoy Das76293462015-11-25 00:42:19 +00004595
Craig Topperc1892ec2019-01-31 17:23:29 +00004596 CallBase *NewCall;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004597 if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
Craig Topperc1892ec2019-01-31 17:23:29 +00004598 NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(),
4599 II->getUnwindDest(), Args, OpBundles);
Craig Topper784929d2019-02-08 20:48:56 +00004600 } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
4601 NewCall = Builder.CreateCallBr(Callee, CBI->getDefaultDest(),
4602 CBI->getIndirectDests(), Args, OpBundles);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004603 } else {
Craig Topperc1892ec2019-01-31 17:23:29 +00004604 NewCall = Builder.CreateCall(Callee, Args, OpBundles);
4605 cast<CallInst>(NewCall)->setTailCallKind(
4606 cast<CallInst>(Caller)->getTailCallKind());
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004607 }
Craig Topperc1892ec2019-01-31 17:23:29 +00004608 NewCall->takeName(Caller);
4609 NewCall->setCallingConv(Call.getCallingConv());
4610 NewCall->setAttributes(NewCallerPAL);
Reid Kleckner257cb4e2017-04-13 20:26:38 +00004611
4612 // Preserve the weight metadata for the new call instruction. The metadata
4613 // is used by SamplePGO to check callsite's hotness.
4614 uint64_t W;
4615 if (Caller->extractProfTotalWeight(W))
Craig Topperc1892ec2019-01-31 17:23:29 +00004616 NewCall->setProfWeight(W);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004617
4618 // Insert a cast of the return type as necessary.
Craig Topperc1892ec2019-01-31 17:23:29 +00004619 Instruction *NC = NewCall;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004620 Value *NV = NC;
4621 if (OldRetTy != NV->getType() && !Caller->use_empty()) {
4622 if (!NV->getType()->isVoidTy()) {
David Majnemer9b6b8222015-01-06 08:41:31 +00004623 NV = NC = CastInst::CreateBitOrPointerCast(NC, OldRetTy);
Eli Friedman35211c62011-05-27 00:19:40 +00004624 NC->setDebugLoc(Caller->getDebugLoc());
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004625
Craig Topper784929d2019-02-08 20:48:56 +00004626 // If this is an invoke/callbr instruction, we should insert it after the
4627 // first non-phi instruction in the normal successor block.
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004628 if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
Bill Wendling07efd6f2011-08-25 01:08:34 +00004629 BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004630 InsertNewInstBefore(NC, *I);
Craig Topper784929d2019-02-08 20:48:56 +00004631 } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
4632 BasicBlock::iterator I = CBI->getDefaultDest()->getFirstInsertionPt();
4633 InsertNewInstBefore(NC, *I);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004634 } else {
Chris Lattner73989652010-12-20 08:25:06 +00004635 // Otherwise, it's a call, just insert cast right after the call.
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004636 InsertNewInstBefore(NC, *Caller);
4637 }
4638 Worklist.AddUsersToWorkList(*Caller);
4639 } else {
4640 NV = UndefValue::get(Caller->getType());
4641 }
4642 }
4643
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004644 if (!Caller->use_empty())
Sanjay Patel4b198802016-02-01 22:23:39 +00004645 replaceInstUsesWith(*Caller, NV);
Frederic Rissc1892e22014-10-23 04:08:42 +00004646 else if (Caller->hasValueHandle()) {
4647 if (OldRetTy == NV->getType())
4648 ValueHandleBase::ValueIsRAUWd(Caller, NV);
4649 else
4650 // We cannot call ValueIsRAUWd with a different type, and the
4651 // actual tracked value will disappear.
4652 ValueHandleBase::ValueIsDeleted(Caller);
4653 }
Eli Friedmanb9ed18f2011-05-18 00:32:01 +00004654
Sanjay Patel4b198802016-02-01 22:23:39 +00004655 eraseInstFromFunction(*Caller);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004656 return true;
4657}
4658
Sanjay Patelcd4377c2016-01-20 22:24:38 +00004659/// Turn a call to a function created by init_trampoline / adjust_trampoline
4660/// intrinsic pair into a direct call to the underlying function.
Duncan Sandsa0984362011-09-06 13:37:06 +00004661Instruction *
Craig Topperc1892ec2019-01-31 17:23:29 +00004662InstCombiner::transformCallThroughTrampoline(CallBase &Call,
4663 IntrinsicInst &Tramp) {
4664 Value *Callee = Call.getCalledValue();
James Y Knight291f7912019-02-01 20:44:54 +00004665 Type *CalleeTy = Callee->getType();
4666 FunctionType *FTy = Call.getFunctionType();
Craig Topperc1892ec2019-01-31 17:23:29 +00004667 AttributeList Attrs = Call.getAttributes();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004668
4669 // If the call already has the 'nest' attribute somewhere then give up -
4670 // otherwise 'nest' would occur twice after splicing in the chain.
Bill Wendling6e95ae82012-12-31 00:49:59 +00004671 if (Attrs.hasAttrSomewhere(Attribute::Nest))
Craig Topperf40110f2014-04-25 05:29:35 +00004672 return nullptr;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004673
Craig Topperc1892ec2019-01-31 17:23:29 +00004674 Function *NestF = cast<Function>(Tramp.getArgOperand(1)->stripPointerCasts());
James Y Knight291f7912019-02-01 20:44:54 +00004675 FunctionType *NestFTy = NestF->getFunctionType();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004676
Reid Klecknereb9dd5b2017-04-10 23:31:05 +00004677 AttributeList NestAttrs = NestF->getAttributes();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004678 if (!NestAttrs.isEmpty()) {
Reid Klecknerf021fab2017-04-13 23:12:13 +00004679 unsigned NestArgNo = 0;
Craig Topperf40110f2014-04-25 05:29:35 +00004680 Type *NestTy = nullptr;
Reid Klecknerc2cb5602017-04-12 00:38:00 +00004681 AttributeSet NestAttr;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004682
4683 // Look for a parameter marked with the 'nest' attribute.
4684 for (FunctionType::param_iterator I = NestFTy->param_begin(),
Reid Klecknerf021fab2017-04-13 23:12:13 +00004685 E = NestFTy->param_end();
4686 I != E; ++NestArgNo, ++I) {
4687 AttributeSet AS = NestAttrs.getParamAttributes(NestArgNo);
4688 if (AS.hasAttribute(Attribute::Nest)) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004689 // Record the parameter type and any other attributes.
4690 NestTy = *I;
Reid Klecknerf021fab2017-04-13 23:12:13 +00004691 NestAttr = AS;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004692 break;
4693 }
Reid Klecknerf021fab2017-04-13 23:12:13 +00004694 }
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004695
4696 if (NestTy) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004697 std::vector<Value*> NewArgs;
Reid Kleckner7f720332017-04-13 00:58:09 +00004698 std::vector<AttributeSet> NewArgAttrs;
Craig Topperc1892ec2019-01-31 17:23:29 +00004699 NewArgs.reserve(Call.arg_size() + 1);
4700 NewArgAttrs.reserve(Call.arg_size());
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004701
4702 // Insert the nest argument into the call argument list, which may
4703 // mean appending it. Likewise for attributes.
4704
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004705 {
Reid Klecknerf021fab2017-04-13 23:12:13 +00004706 unsigned ArgNo = 0;
Craig Topperc1892ec2019-01-31 17:23:29 +00004707 auto I = Call.arg_begin(), E = Call.arg_end();
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004708 do {
Reid Klecknerf021fab2017-04-13 23:12:13 +00004709 if (ArgNo == NestArgNo) {
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004710 // Add the chain argument and attributes.
Craig Topperc1892ec2019-01-31 17:23:29 +00004711 Value *NestVal = Tramp.getArgOperand(2);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004712 if (NestVal->getType() != NestTy)
Craig Topperbb4069e2017-07-07 23:16:26 +00004713 NestVal = Builder.CreateBitCast(NestVal, NestTy, "nest");
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004714 NewArgs.push_back(NestVal);
Reid Kleckner7f720332017-04-13 00:58:09 +00004715 NewArgAttrs.push_back(NestAttr);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004716 }
4717
4718 if (I == E)
4719 break;
4720
4721 // Add the original argument and attributes.
4722 NewArgs.push_back(*I);
Reid Klecknerf021fab2017-04-13 23:12:13 +00004723 NewArgAttrs.push_back(Attrs.getParamAttributes(ArgNo));
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004724
Reid Klecknerf021fab2017-04-13 23:12:13 +00004725 ++ArgNo;
Richard Trieu7a083812016-02-18 22:09:30 +00004726 ++I;
Eugene Zelenkocdc71612016-08-11 17:20:18 +00004727 } while (true);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004728 }
4729
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004730 // The trampoline may have been bitcast to a bogus type (FTy).
4731 // Handle this by synthesizing a new function type, equal to FTy
4732 // with the chain parameter inserted.
4733
Jay Foadb804a2b2011-07-12 14:06:48 +00004734 std::vector<Type*> NewTypes;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004735 NewTypes.reserve(FTy->getNumParams()+1);
4736
4737 // Insert the chain's type into the list of parameter types, which may
4738 // mean appending it.
4739 {
Reid Klecknerf021fab2017-04-13 23:12:13 +00004740 unsigned ArgNo = 0;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004741 FunctionType::param_iterator I = FTy->param_begin(),
4742 E = FTy->param_end();
4743
4744 do {
Reid Klecknerf021fab2017-04-13 23:12:13 +00004745 if (ArgNo == NestArgNo)
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004746 // Add the chain's type.
4747 NewTypes.push_back(NestTy);
4748
4749 if (I == E)
4750 break;
4751
4752 // Add the original type.
4753 NewTypes.push_back(*I);
4754
Reid Klecknerf021fab2017-04-13 23:12:13 +00004755 ++ArgNo;
Richard Trieu7a083812016-02-18 22:09:30 +00004756 ++I;
Eugene Zelenkocdc71612016-08-11 17:20:18 +00004757 } while (true);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004758 }
4759
4760 // Replace the trampoline call with a direct call. Let the generic
4761 // code sort out any function type mismatches.
Jim Grosbach7815f562012-02-03 00:07:04 +00004762 FunctionType *NewFTy = FunctionType::get(FTy->getReturnType(), NewTypes,
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004763 FTy->isVarArg());
4764 Constant *NewCallee =
4765 NestF->getType() == PointerType::getUnqual(NewFTy) ?
Jim Grosbach7815f562012-02-03 00:07:04 +00004766 NestF : ConstantExpr::getBitCast(NestF,
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004767 PointerType::getUnqual(NewFTy));
Reid Kleckner7f720332017-04-13 00:58:09 +00004768 AttributeList NewPAL =
4769 AttributeList::get(FTy->getContext(), Attrs.getFnAttributes(),
4770 Attrs.getRetAttributes(), NewArgAttrs);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004771
David Majnemer231a68c2016-04-29 08:07:20 +00004772 SmallVector<OperandBundleDef, 1> OpBundles;
Craig Topperc1892ec2019-01-31 17:23:29 +00004773 Call.getOperandBundlesAsDefs(OpBundles);
David Majnemer231a68c2016-04-29 08:07:20 +00004774
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004775 Instruction *NewCaller;
Craig Topperc1892ec2019-01-31 17:23:29 +00004776 if (InvokeInst *II = dyn_cast<InvokeInst>(&Call)) {
James Y Knight7976eb52019-02-01 20:43:25 +00004777 NewCaller = InvokeInst::Create(NewFTy, NewCallee,
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004778 II->getNormalDest(), II->getUnwindDest(),
David Majnemer231a68c2016-04-29 08:07:20 +00004779 NewArgs, OpBundles);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004780 cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv());
4781 cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);
Craig Topper784929d2019-02-08 20:48:56 +00004782 } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&Call)) {
4783 NewCaller =
4784 CallBrInst::Create(NewFTy, NewCallee, CBI->getDefaultDest(),
4785 CBI->getIndirectDests(), NewArgs, OpBundles);
4786 cast<CallBrInst>(NewCaller)->setCallingConv(CBI->getCallingConv());
4787 cast<CallBrInst>(NewCaller)->setAttributes(NewPAL);
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004788 } else {
James Y Knight7976eb52019-02-01 20:43:25 +00004789 NewCaller = CallInst::Create(NewFTy, NewCallee, NewArgs, OpBundles);
David Majnemerd5648c72016-11-25 22:35:09 +00004790 cast<CallInst>(NewCaller)->setTailCallKind(
Craig Topperc1892ec2019-01-31 17:23:29 +00004791 cast<CallInst>(Call).getTailCallKind());
David Majnemerd5648c72016-11-25 22:35:09 +00004792 cast<CallInst>(NewCaller)->setCallingConv(
Craig Topperc1892ec2019-01-31 17:23:29 +00004793 cast<CallInst>(Call).getCallingConv());
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004794 cast<CallInst>(NewCaller)->setAttributes(NewPAL);
4795 }
Craig Topperc1892ec2019-01-31 17:23:29 +00004796 NewCaller->setDebugLoc(Call.getDebugLoc());
Eli Friedman49346012011-05-18 19:57:14 +00004797
4798 return NewCaller;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004799 }
4800 }
4801
4802 // Replace the trampoline call with a direct call. Since there is no 'nest'
4803 // parameter, there is no need to adjust the argument list. Let the generic
4804 // code sort out any function type mismatches.
James Y Knight291f7912019-02-01 20:44:54 +00004805 Constant *NewCallee = ConstantExpr::getBitCast(NestF, CalleeTy);
4806 Call.setCalledFunction(FTy, NewCallee);
Craig Topperc1892ec2019-01-31 17:23:29 +00004807 return &Call;
Chris Lattner7a9e47a2010-01-05 07:32:13 +00004808}