blob: 3477c087967f03c245a70eacfb71f815d2816f6e [file] [log] [blame]
Chris Lattner17079fc2009-12-28 21:28:46 +00001//===---- IRBuilder.cpp - Builder for LLVM Instrs -------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the IRBuilder class, which is used as a convenient way
11// to create LLVM instructions with a consistent and simplified interface.
12//
13//===----------------------------------------------------------------------===//
14
Chandler Carruth9fb823b2013-01-02 11:36:10 +000015#include "llvm/IR/Function.h"
16#include "llvm/IR/GlobalVariable.h"
17#include "llvm/IR/IRBuilder.h"
18#include "llvm/IR/Intrinsics.h"
19#include "llvm/IR/LLVMContext.h"
Pat Gavlincc0431d2015-05-08 18:07:42 +000020#include "llvm/IR/Statepoint.h"
Chris Lattner17079fc2009-12-28 21:28:46 +000021using namespace llvm;
22
23/// CreateGlobalString - Make a new global variable with an initializer that
Dan Gohman97c59022010-02-10 20:04:19 +000024/// has array of i8 type filled in with the nul terminated string value
Chris Lattner17079fc2009-12-28 21:28:46 +000025/// specified. If Name is specified, it is the name of the global variable
26/// created.
David Blaikieaa41cd52015-04-03 21:33:42 +000027GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
Tobias Grossercdb89142015-06-19 02:12:07 +000028 const Twine &Name,
29 unsigned AddressSpace) {
Chris Lattnercf9e8f62012-02-05 02:29:43 +000030 Constant *StrConstant = ConstantDataArray::getString(Context, Str);
Chris Lattner17079fc2009-12-28 21:28:46 +000031 Module &M = *BB->getParent()->getParent();
32 GlobalVariable *GV = new GlobalVariable(M, StrConstant->getType(),
Benjamin Kramerf1fd6e32011-12-22 14:22:14 +000033 true, GlobalValue::PrivateLinkage,
Tobias Grossercdb89142015-06-19 02:12:07 +000034 StrConstant, Name, nullptr,
35 GlobalVariable::NotThreadLocal,
36 AddressSpace);
Peter Collingbourne96efdd62016-06-14 21:01:22 +000037 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
Chris Lattner17079fc2009-12-28 21:28:46 +000038 return GV;
39}
Chris Lattner7ef1cac2009-12-28 21:45:40 +000040
Chris Lattnerb1907b22011-07-12 04:14:22 +000041Type *IRBuilderBase::getCurrentFunctionReturnType() const {
Chris Lattner49f9f762009-12-28 21:50:56 +000042 assert(BB && BB->getParent() && "No current function!");
43 return BB->getParent()->getReturnType();
44}
Chris Lattner143a07c2010-12-26 22:49:25 +000045
46Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
Chris Lattner229907c2011-07-18 04:54:35 +000047 PointerType *PT = cast<PointerType>(Ptr->getType());
Chris Lattner143a07c2010-12-26 22:49:25 +000048 if (PT->getElementType()->isIntegerTy(8))
49 return Ptr;
50
51 // Otherwise, we need to insert a bitcast.
52 PT = getInt8PtrTy(PT->getAddressSpace());
53 BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
54 BB->getInstList().insert(InsertPt, BCI);
55 SetInstDebugLocation(BCI);
56 return BCI;
57}
58
Jay Foad5bd375a2011-07-15 08:37:34 +000059static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
Philip Reames4750dd12014-12-30 05:55:58 +000060 IRBuilderBase *Builder,
61 const Twine& Name="") {
62 CallInst *CI = CallInst::Create(Callee, Ops, Name);
Chris Lattner143a07c2010-12-26 22:49:25 +000063 Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
64 Builder->SetInstDebugLocation(CI);
65 return CI;
66}
67
Sanjoy Dasabe1c682015-05-06 23:53:09 +000068static InvokeInst *createInvokeHelper(Value *Invokee, BasicBlock *NormalDest,
69 BasicBlock *UnwindDest,
70 ArrayRef<Value *> Ops,
71 IRBuilderBase *Builder,
72 const Twine &Name = "") {
73 InvokeInst *II =
74 InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
75 Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
76 II);
77 Builder->SetInstDebugLocation(II);
78 return II;
79}
80
Chris Lattner143a07c2010-12-26 22:49:25 +000081CallInst *IRBuilderBase::
Pete Cooper67cf9a72015-11-19 05:56:52 +000082CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
Hal Finkel94146652014-07-24 14:25:39 +000083 bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
84 MDNode *NoAliasTag) {
Chris Lattner143a07c2010-12-26 22:49:25 +000085 Ptr = getCastedInt8PtrValue(Ptr);
Pete Cooper67cf9a72015-11-19 05:56:52 +000086 Value *Ops[] = { Ptr, Val, Size, getInt32(Align), getInt1(isVolatile) };
Jay Foadb804a2b2011-07-12 14:06:48 +000087 Type *Tys[] = { Ptr->getType(), Size->getType() };
Chris Lattner143a07c2010-12-26 22:49:25 +000088 Module *M = BB->getParent()->getParent();
Benjamin Kramere6e19332011-07-14 17:45:39 +000089 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
Chris Lattner143a07c2010-12-26 22:49:25 +000090
Jay Foad5bd375a2011-07-15 08:37:34 +000091 CallInst *CI = createCallHelper(TheFn, Ops, this);
Chris Lattner143a07c2010-12-26 22:49:25 +000092
93 // Set the TBAA info if present.
94 if (TBAATag)
95 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
Hal Finkel94146652014-07-24 14:25:39 +000096
97 if (ScopeTag)
98 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
99
100 if (NoAliasTag)
101 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
102
Chris Lattner143a07c2010-12-26 22:49:25 +0000103 return CI;
104}
105
106CallInst *IRBuilderBase::
Pete Cooper67cf9a72015-11-19 05:56:52 +0000107CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align,
Hal Finkel94146652014-07-24 14:25:39 +0000108 bool isVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
109 MDNode *ScopeTag, MDNode *NoAliasTag) {
Chris Lattner143a07c2010-12-26 22:49:25 +0000110 Dst = getCastedInt8PtrValue(Dst);
111 Src = getCastedInt8PtrValue(Src);
112
Pete Cooper67cf9a72015-11-19 05:56:52 +0000113 Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) };
Jay Foadb804a2b2011-07-12 14:06:48 +0000114 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
Chris Lattner143a07c2010-12-26 22:49:25 +0000115 Module *M = BB->getParent()->getParent();
Benjamin Kramere6e19332011-07-14 17:45:39 +0000116 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
Chris Lattner143a07c2010-12-26 22:49:25 +0000117
Jay Foad5bd375a2011-07-15 08:37:34 +0000118 CallInst *CI = createCallHelper(TheFn, Ops, this);
Chris Lattner143a07c2010-12-26 22:49:25 +0000119
120 // Set the TBAA info if present.
121 if (TBAATag)
122 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
Dan Gohman099727f2012-09-26 22:17:14 +0000123
124 // Set the TBAA Struct info if present.
125 if (TBAAStructTag)
126 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
Hal Finkel94146652014-07-24 14:25:39 +0000127
128 if (ScopeTag)
129 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
130
131 if (NoAliasTag)
132 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
133
Chris Lattner143a07c2010-12-26 22:49:25 +0000134 return CI;
135}
136
137CallInst *IRBuilderBase::
Pete Cooper67cf9a72015-11-19 05:56:52 +0000138CreateMemMove(Value *Dst, Value *Src, Value *Size, unsigned Align,
Hal Finkel94146652014-07-24 14:25:39 +0000139 bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
140 MDNode *NoAliasTag) {
Chris Lattner143a07c2010-12-26 22:49:25 +0000141 Dst = getCastedInt8PtrValue(Dst);
142 Src = getCastedInt8PtrValue(Src);
143
Pete Cooper67cf9a72015-11-19 05:56:52 +0000144 Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) };
Jay Foadb804a2b2011-07-12 14:06:48 +0000145 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
Chris Lattner143a07c2010-12-26 22:49:25 +0000146 Module *M = BB->getParent()->getParent();
Benjamin Kramere6e19332011-07-14 17:45:39 +0000147 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
Chris Lattner143a07c2010-12-26 22:49:25 +0000148
Jay Foad5bd375a2011-07-15 08:37:34 +0000149 CallInst *CI = createCallHelper(TheFn, Ops, this);
Chris Lattner143a07c2010-12-26 22:49:25 +0000150
151 // Set the TBAA info if present.
152 if (TBAATag)
153 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
Hal Finkel94146652014-07-24 14:25:39 +0000154
155 if (ScopeTag)
156 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
157
158 if (NoAliasTag)
159 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
160
Chris Lattner143a07c2010-12-26 22:49:25 +0000161 return CI;
162}
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000163
Amara Emersoncf9daa32017-05-09 10:43:25 +0000164static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
165 Value *Src) {
166 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
167 Value *Ops[] = {Src};
168 Type *Tys[] = { Src->getType()->getVectorElementType(), Src->getType() };
169 auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
170 return createCallHelper(Decl, Ops, Builder);
171}
172
173CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
174 Module *M = GetInsertBlock()->getParent()->getParent();
175 Value *Ops[] = {Acc, Src};
176 Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
177 Src->getType()};
178 auto Decl = Intrinsic::getDeclaration(
179 M, Intrinsic::experimental_vector_reduce_fadd, Tys);
180 return createCallHelper(Decl, Ops, this);
181}
182
183CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
184 Module *M = GetInsertBlock()->getParent()->getParent();
185 Value *Ops[] = {Acc, Src};
186 Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
187 Src->getType()};
188 auto Decl = Intrinsic::getDeclaration(
189 M, Intrinsic::experimental_vector_reduce_fmul, Tys);
190 return createCallHelper(Decl, Ops, this);
191}
192
193CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
194 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
195 Src);
196}
197
198CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
199 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
200 Src);
201}
202
203CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
204 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
205 Src);
206}
207
208CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
209 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
210 Src);
211}
212
213CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
214 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
215 Src);
216}
217
218CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
219 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
220 : Intrinsic::experimental_vector_reduce_umax;
221 return getReductionIntrinsic(this, ID, Src);
222}
223
224CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
225 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
226 : Intrinsic::experimental_vector_reduce_umin;
227 return getReductionIntrinsic(this, ID, Src);
228}
229
230CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
231 auto Rdx = getReductionIntrinsic(
232 this, Intrinsic::experimental_vector_reduce_fmax, Src);
233 if (NoNaN) {
234 FastMathFlags FMF;
235 FMF.setNoNaNs();
236 Rdx->setFastMathFlags(FMF);
237 }
238 return Rdx;
239}
240
241CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
242 auto Rdx = getReductionIntrinsic(
243 this, Intrinsic::experimental_vector_reduce_fmin, Src);
244 if (NoNaN) {
245 FastMathFlags FMF;
246 FMF.setNoNaNs();
247 Rdx->setFastMathFlags(FMF);
248 }
249 return Rdx;
250}
251
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000252CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
253 assert(isa<PointerType>(Ptr->getType()) &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000254 "lifetime.start only applies to pointers.");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000255 Ptr = getCastedInt8PtrValue(Ptr);
256 if (!Size)
257 Size = getInt64(-1);
258 else
259 assert(Size->getType() == getInt64Ty() &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000260 "lifetime.start requires the size to be an i64");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000261 Value *Ops[] = { Size, Ptr };
262 Module *M = BB->getParent()->getParent();
Matt Arsenaultf10061e2017-04-10 20:18:21 +0000263 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_start,
264 { Ptr->getType() });
Jay Foad5bd375a2011-07-15 08:37:34 +0000265 return createCallHelper(TheFn, Ops, this);
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000266}
267
268CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
269 assert(isa<PointerType>(Ptr->getType()) &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000270 "lifetime.end only applies to pointers.");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000271 Ptr = getCastedInt8PtrValue(Ptr);
272 if (!Size)
273 Size = getInt64(-1);
274 else
275 assert(Size->getType() == getInt64Ty() &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000276 "lifetime.end requires the size to be an i64");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000277 Value *Ops[] = { Size, Ptr };
278 Module *M = BB->getParent()->getParent();
Matt Arsenaultf10061e2017-04-10 20:18:21 +0000279 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_end,
280 { Ptr->getType() });
Jay Foad5bd375a2011-07-15 08:37:34 +0000281 return createCallHelper(TheFn, Ops, this);
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000282}
Hal Finkel6f814db2014-10-15 23:44:22 +0000283
Anna Thomas58d11922016-07-22 20:57:23 +0000284CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
285
286 assert(isa<PointerType>(Ptr->getType()) &&
287 "invariant.start only applies to pointers.");
288 Ptr = getCastedInt8PtrValue(Ptr);
289 if (!Size)
290 Size = getInt64(-1);
291 else
292 assert(Size->getType() == getInt64Ty() &&
293 "invariant.start requires the size to be an i64");
294
295 Value *Ops[] = {Size, Ptr};
296 // Fill in the single overloaded type: memory object type.
297 Type *ObjectPtr[1] = {Ptr->getType()};
298 Module *M = BB->getParent()->getParent();
299 Value *TheFn =
300 Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
301 return createCallHelper(TheFn, Ops, this);
302}
303
Hal Finkel6f814db2014-10-15 23:44:22 +0000304CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
305 assert(Cond->getType() == getInt1Ty() &&
306 "an assumption condition must be of type i1");
307
308 Value *Ops[] = { Cond };
309 Module *M = BB->getParent()->getParent();
310 Value *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
311 return createCallHelper(FnAssume, Ops, this);
312}
313
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000314/// \brief Create a call to a Masked Load intrinsic.
315/// \p Ptr - base pointer for the load
316/// \p Align - alignment of the source location
317/// \p Mask - vector of booleans which indicates what vector lanes should
318/// be accessed in memory
319/// \p PassThru - pass-through value that is used to fill the masked-off lanes
320/// of the result
321/// \p Name - name of the result variable
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000322CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
323 Value *Mask, Value *PassThru,
324 const Twine &Name) {
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000325 PointerType *PtrTy = cast<PointerType>(Ptr->getType());
326 Type *DataTy = PtrTy->getElementType();
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000327 assert(DataTy->isVectorTy() && "Ptr should point to a vector");
328 if (!PassThru)
329 PassThru = UndefValue::get(DataTy);
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000330 Type *OverloadedTypes[] = { DataTy, PtrTy };
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000331 Value *Ops[] = { Ptr, getInt32(Align), Mask, PassThru};
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000332 return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
333 OverloadedTypes, Name);
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000334}
335
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000336/// \brief Create a call to a Masked Store intrinsic.
337/// \p Val - data to be stored,
338/// \p Ptr - base pointer for the store
339/// \p Align - alignment of the destination location
340/// \p Mask - vector of booleans which indicates what vector lanes should
341/// be accessed in memory
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000342CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
343 unsigned Align, Value *Mask) {
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000344 PointerType *PtrTy = cast<PointerType>(Ptr->getType());
345 Type *DataTy = PtrTy->getElementType();
346 assert(DataTy->isVectorTy() && "Ptr should point to a vector");
347 Type *OverloadedTypes[] = { DataTy, PtrTy };
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000348 Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000349 return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000350}
351
352/// Create a call to a Masked intrinsic, with given intrinsic Id,
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000353/// an array of operands - Ops, and an array of overloaded types -
354/// OverloadedTypes.
Pete Cooper9e1d3352015-05-20 17:16:39 +0000355CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000356 ArrayRef<Value *> Ops,
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000357 ArrayRef<Type *> OverloadedTypes,
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000358 const Twine &Name) {
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000359 Module *M = BB->getParent()->getParent();
Pete Cooper9e1d3352015-05-20 17:16:39 +0000360 Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000361 return createCallHelper(TheFn, Ops, this, Name);
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000362}
Philip Reames4750dd12014-12-30 05:55:58 +0000363
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000364/// \brief Create a call to a Masked Gather intrinsic.
365/// \p Ptrs - vector of pointers for loading
366/// \p Align - alignment for one element
367/// \p Mask - vector of booleans which indicates what vector lanes should
368/// be accessed in memory
369/// \p PassThru - pass-through value that is used to fill the masked-off lanes
370/// of the result
371/// \p Name - name of the result variable
372CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
373 Value *Mask, Value *PassThru,
374 const Twine& Name) {
375 auto PtrsTy = cast<VectorType>(Ptrs->getType());
376 auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
377 unsigned NumElts = PtrsTy->getVectorNumElements();
378 Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
379
380 if (!Mask)
381 Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
382 NumElts));
383
Elad Cohenef5798a2017-05-03 12:28:54 +0000384 Type *OverloadedTypes[] = {DataTy, PtrsTy};
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000385 Value * Ops[] = {Ptrs, getInt32(Align), Mask, UndefValue::get(DataTy)};
386
387 // We specify only one type when we create this intrinsic. Types of other
388 // arguments are derived from this type.
Elad Cohenef5798a2017-05-03 12:28:54 +0000389 return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
390 Name);
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000391}
392
393/// \brief Create a call to a Masked Scatter intrinsic.
394/// \p Data - data to be stored,
395/// \p Ptrs - the vector of pointers, where the \p Data elements should be
396/// stored
397/// \p Align - alignment for one element
398/// \p Mask - vector of booleans which indicates what vector lanes should
399/// be accessed in memory
400CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
401 unsigned Align, Value *Mask) {
402 auto PtrsTy = cast<VectorType>(Ptrs->getType());
403 auto DataTy = cast<VectorType>(Data->getType());
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000404 unsigned NumElts = PtrsTy->getVectorNumElements();
405
Tim Northover5a1a56c2016-02-17 21:16:59 +0000406#ifndef NDEBUG
407 auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000408 assert(NumElts == DataTy->getVectorNumElements() &&
Tim Northover5a1a56c2016-02-17 21:16:59 +0000409 PtrTy->getElementType() == DataTy->getElementType() &&
410 "Incompatible pointer and data types");
411#endif
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000412
413 if (!Mask)
414 Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
415 NumElts));
Elad Cohenef5798a2017-05-03 12:28:54 +0000416
417 Type *OverloadedTypes[] = {DataTy, PtrsTy};
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000418 Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
419
420 // We specify only one type when we create this intrinsic. Types of other
421 // arguments are derived from this type.
Elad Cohenef5798a2017-05-03 12:28:54 +0000422 return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000423}
424
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000425template <typename T0, typename T1, typename T2, typename T3>
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000426static std::vector<Value *>
427getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000428 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
429 ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
430 ArrayRef<T3> GCArgs) {
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000431 std::vector<Value *> Args;
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000432 Args.push_back(B.getInt64(ID));
433 Args.push_back(B.getInt32(NumPatchBytes));
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000434 Args.push_back(ActualCallee);
435 Args.push_back(B.getInt32(CallArgs.size()));
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000436 Args.push_back(B.getInt32(Flags));
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000437 Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000438 Args.push_back(B.getInt32(TransitionArgs.size()));
439 Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000440 Args.push_back(B.getInt32(DeoptArgs.size()));
441 Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
442 Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
443
444 return Args;
445}
446
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000447template <typename T0, typename T1, typename T2, typename T3>
448static CallInst *CreateGCStatepointCallCommon(
449 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000450 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000451 ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
452 const Twine &Name) {
Sanjoy Das63245b52015-05-06 02:36:34 +0000453 // Extract out the type of the callee.
454 PointerType *FuncPtrType = cast<PointerType>(ActualCallee->getType());
455 assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
456 "actual callee must be a callable value");
Philip Reames4750dd12014-12-30 05:55:58 +0000457
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000458 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
Sanjoy Das63245b52015-05-06 02:36:34 +0000459 // Fill in the one generic type'd argument (the function is also vararg)
460 Type *ArgTypes[] = { FuncPtrType };
461 Function *FnStatepoint =
462 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
463 ArgTypes);
Philip Reames4750dd12014-12-30 05:55:58 +0000464
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000465 std::vector<llvm::Value *> Args =
466 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
467 CallArgs, TransitionArgs, DeoptArgs, GCArgs);
468 return createCallHelper(FnStatepoint, Args, Builder, Name);
469}
470
471CallInst *IRBuilderBase::CreateGCStatepointCall(
472 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
473 ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
474 ArrayRef<Value *> GCArgs, const Twine &Name) {
475 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000476 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
477 CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000478}
479
480CallInst *IRBuilderBase::CreateGCStatepointCall(
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000481 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
482 ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000483 ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
484 return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
485 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
486 DeoptArgs, GCArgs, Name);
Philip Reames4750dd12014-12-30 05:55:58 +0000487}
488
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000489CallInst *IRBuilderBase::CreateGCStatepointCall(
490 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
491 ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
492 ArrayRef<Value *> GCArgs, const Twine &Name) {
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000493 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000494 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
495 CallArgs, None, DeoptArgs, GCArgs, Name);
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000496}
497
498template <typename T0, typename T1, typename T2, typename T3>
499static InvokeInst *CreateGCStatepointInvokeCommon(
500 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
501 Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000502 uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000503 ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
504 // Extract out the type of the callee.
505 PointerType *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
506 assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
507 "actual callee must be a callable value");
508
509 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
510 // Fill in the one generic type'd argument (the function is also vararg)
511 Function *FnStatepoint = Intrinsic::getDeclaration(
512 M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
513
514 std::vector<llvm::Value *> Args =
515 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
516 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
517 return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
518 Name);
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000519}
520
521InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000522 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
523 BasicBlock *NormalDest, BasicBlock *UnwindDest,
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000524 ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
525 ArrayRef<Value *> GCArgs, const Twine &Name) {
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000526 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
527 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000528 uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000529 DeoptArgs, GCArgs, Name);
530}
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000531
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000532InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
533 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000534 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000535 ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
536 ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
537 return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
538 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
539 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000540}
541
542InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000543 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
544 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
545 ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000546 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
547 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000548 uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
549 Name);
Ramkumar Ramachandra3408f3e2015-02-26 00:35:56 +0000550}
551
Philip Reames4750dd12014-12-30 05:55:58 +0000552CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
553 Type *ResultType,
554 const Twine &Name) {
Ramkumar Ramachandra75a4f352015-01-22 20:14:38 +0000555 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
Philip Reames4750dd12014-12-30 05:55:58 +0000556 Module *M = BB->getParent()->getParent();
557 Type *Types[] = {ResultType};
558 Value *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
559
560 Value *Args[] = {Statepoint};
561 return createCallHelper(FnGCResult, Args, this, Name);
562}
563
564CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
565 int BaseOffset,
566 int DerivedOffset,
567 Type *ResultType,
568 const Twine &Name) {
569 Module *M = BB->getParent()->getParent();
570 Type *Types[] = {ResultType};
571 Value *FnGCRelocate =
572 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
573
574 Value *Args[] = {Statepoint,
575 getInt32(BaseOffset),
576 getInt32(DerivedOffset)};
577 return createCallHelper(FnGCRelocate, Args, this, Name);
578}
Matt Arsenaultcdb468c2017-02-27 23:08:49 +0000579
580CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID,
581 Value *LHS, Value *RHS,
582 const Twine &Name) {
583 Module *M = BB->getParent()->getParent();
584 Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
585 return createCallHelper(Fn, { LHS, RHS }, this, Name);
586}