blob: 7572d0c6b3bc49f391539840f9c987e43f158720 [file] [log] [blame]
Chris Lattner17079fc2009-12-28 21:28:46 +00001//===---- IRBuilder.cpp - Builder for LLVM Instrs -------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the IRBuilder class, which is used as a convenient way
11// to create LLVM instructions with a consistent and simplified interface.
12//
13//===----------------------------------------------------------------------===//
14
Chandler Carruth9fb823b2013-01-02 11:36:10 +000015#include "llvm/IR/Function.h"
16#include "llvm/IR/GlobalVariable.h"
17#include "llvm/IR/IRBuilder.h"
18#include "llvm/IR/Intrinsics.h"
19#include "llvm/IR/LLVMContext.h"
Pat Gavlincc0431d2015-05-08 18:07:42 +000020#include "llvm/IR/Statepoint.h"
Chris Lattner17079fc2009-12-28 21:28:46 +000021using namespace llvm;
22
23/// CreateGlobalString - Make a new global variable with an initializer that
Dan Gohman97c59022010-02-10 20:04:19 +000024/// has array of i8 type filled in with the nul terminated string value
Chris Lattner17079fc2009-12-28 21:28:46 +000025/// specified. If Name is specified, it is the name of the global variable
26/// created.
David Blaikieaa41cd52015-04-03 21:33:42 +000027GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
Tobias Grossercdb89142015-06-19 02:12:07 +000028 const Twine &Name,
29 unsigned AddressSpace) {
Chris Lattnercf9e8f62012-02-05 02:29:43 +000030 Constant *StrConstant = ConstantDataArray::getString(Context, Str);
Chris Lattner17079fc2009-12-28 21:28:46 +000031 Module &M = *BB->getParent()->getParent();
32 GlobalVariable *GV = new GlobalVariable(M, StrConstant->getType(),
Benjamin Kramerf1fd6e32011-12-22 14:22:14 +000033 true, GlobalValue::PrivateLinkage,
Tobias Grossercdb89142015-06-19 02:12:07 +000034 StrConstant, Name, nullptr,
35 GlobalVariable::NotThreadLocal,
36 AddressSpace);
Peter Collingbourne96efdd62016-06-14 21:01:22 +000037 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
Chris Lattner17079fc2009-12-28 21:28:46 +000038 return GV;
39}
Chris Lattner7ef1cac2009-12-28 21:45:40 +000040
Chris Lattnerb1907b22011-07-12 04:14:22 +000041Type *IRBuilderBase::getCurrentFunctionReturnType() const {
Chris Lattner49f9f762009-12-28 21:50:56 +000042 assert(BB && BB->getParent() && "No current function!");
43 return BB->getParent()->getReturnType();
44}
Chris Lattner143a07c2010-12-26 22:49:25 +000045
46Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
Chris Lattner229907c2011-07-18 04:54:35 +000047 PointerType *PT = cast<PointerType>(Ptr->getType());
Chris Lattner143a07c2010-12-26 22:49:25 +000048 if (PT->getElementType()->isIntegerTy(8))
49 return Ptr;
50
51 // Otherwise, we need to insert a bitcast.
52 PT = getInt8PtrTy(PT->getAddressSpace());
53 BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
54 BB->getInstList().insert(InsertPt, BCI);
55 SetInstDebugLocation(BCI);
56 return BCI;
57}
58
Jay Foad5bd375a2011-07-15 08:37:34 +000059static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
Philip Reames4750dd12014-12-30 05:55:58 +000060 IRBuilderBase *Builder,
61 const Twine& Name="") {
62 CallInst *CI = CallInst::Create(Callee, Ops, Name);
Chris Lattner143a07c2010-12-26 22:49:25 +000063 Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
64 Builder->SetInstDebugLocation(CI);
65 return CI;
66}
67
Sanjoy Dasabe1c682015-05-06 23:53:09 +000068static InvokeInst *createInvokeHelper(Value *Invokee, BasicBlock *NormalDest,
69 BasicBlock *UnwindDest,
70 ArrayRef<Value *> Ops,
71 IRBuilderBase *Builder,
72 const Twine &Name = "") {
73 InvokeInst *II =
74 InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
75 Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
76 II);
77 Builder->SetInstDebugLocation(II);
78 return II;
79}
80
Chris Lattner143a07c2010-12-26 22:49:25 +000081CallInst *IRBuilderBase::
Pete Cooper67cf9a72015-11-19 05:56:52 +000082CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
Hal Finkel94146652014-07-24 14:25:39 +000083 bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
84 MDNode *NoAliasTag) {
Chris Lattner143a07c2010-12-26 22:49:25 +000085 Ptr = getCastedInt8PtrValue(Ptr);
Pete Cooper67cf9a72015-11-19 05:56:52 +000086 Value *Ops[] = { Ptr, Val, Size, getInt32(Align), getInt1(isVolatile) };
Jay Foadb804a2b2011-07-12 14:06:48 +000087 Type *Tys[] = { Ptr->getType(), Size->getType() };
Chris Lattner143a07c2010-12-26 22:49:25 +000088 Module *M = BB->getParent()->getParent();
Benjamin Kramere6e19332011-07-14 17:45:39 +000089 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
Chris Lattner143a07c2010-12-26 22:49:25 +000090
Jay Foad5bd375a2011-07-15 08:37:34 +000091 CallInst *CI = createCallHelper(TheFn, Ops, this);
Chris Lattner143a07c2010-12-26 22:49:25 +000092
93 // Set the TBAA info if present.
94 if (TBAATag)
95 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
Hal Finkel94146652014-07-24 14:25:39 +000096
97 if (ScopeTag)
98 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
99
100 if (NoAliasTag)
101 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
102
Chris Lattner143a07c2010-12-26 22:49:25 +0000103 return CI;
104}
105
106CallInst *IRBuilderBase::
Pete Cooper67cf9a72015-11-19 05:56:52 +0000107CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align,
Hal Finkel94146652014-07-24 14:25:39 +0000108 bool isVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
109 MDNode *ScopeTag, MDNode *NoAliasTag) {
Chris Lattner143a07c2010-12-26 22:49:25 +0000110 Dst = getCastedInt8PtrValue(Dst);
111 Src = getCastedInt8PtrValue(Src);
112
Pete Cooper67cf9a72015-11-19 05:56:52 +0000113 Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) };
Jay Foadb804a2b2011-07-12 14:06:48 +0000114 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
Chris Lattner143a07c2010-12-26 22:49:25 +0000115 Module *M = BB->getParent()->getParent();
Benjamin Kramere6e19332011-07-14 17:45:39 +0000116 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
Chris Lattner143a07c2010-12-26 22:49:25 +0000117
Jay Foad5bd375a2011-07-15 08:37:34 +0000118 CallInst *CI = createCallHelper(TheFn, Ops, this);
Chris Lattner143a07c2010-12-26 22:49:25 +0000119
120 // Set the TBAA info if present.
121 if (TBAATag)
122 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
Dan Gohman099727f2012-09-26 22:17:14 +0000123
124 // Set the TBAA Struct info if present.
125 if (TBAAStructTag)
126 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
Hal Finkel94146652014-07-24 14:25:39 +0000127
128 if (ScopeTag)
129 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
130
131 if (NoAliasTag)
132 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
133
Chris Lattner143a07c2010-12-26 22:49:25 +0000134 return CI;
135}
136
137CallInst *IRBuilderBase::
Pete Cooper67cf9a72015-11-19 05:56:52 +0000138CreateMemMove(Value *Dst, Value *Src, Value *Size, unsigned Align,
Hal Finkel94146652014-07-24 14:25:39 +0000139 bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
140 MDNode *NoAliasTag) {
Chris Lattner143a07c2010-12-26 22:49:25 +0000141 Dst = getCastedInt8PtrValue(Dst);
142 Src = getCastedInt8PtrValue(Src);
143
Pete Cooper67cf9a72015-11-19 05:56:52 +0000144 Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) };
Jay Foadb804a2b2011-07-12 14:06:48 +0000145 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
Chris Lattner143a07c2010-12-26 22:49:25 +0000146 Module *M = BB->getParent()->getParent();
Benjamin Kramere6e19332011-07-14 17:45:39 +0000147 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
Chris Lattner143a07c2010-12-26 22:49:25 +0000148
Jay Foad5bd375a2011-07-15 08:37:34 +0000149 CallInst *CI = createCallHelper(TheFn, Ops, this);
Chris Lattner143a07c2010-12-26 22:49:25 +0000150
151 // Set the TBAA info if present.
152 if (TBAATag)
153 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
Hal Finkel94146652014-07-24 14:25:39 +0000154
155 if (ScopeTag)
156 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
157
158 if (NoAliasTag)
159 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
160
Chris Lattner143a07c2010-12-26 22:49:25 +0000161 return CI;
162}
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000163
Amara Emersoncf9daa32017-05-09 10:43:25 +0000164static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
165 Value *Src) {
166 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
167 Value *Ops[] = {Src};
168 Type *Tys[] = { Src->getType()->getVectorElementType(), Src->getType() };
169 auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
170 return createCallHelper(Decl, Ops, Builder);
171}
172
173CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
174 Module *M = GetInsertBlock()->getParent()->getParent();
175 Value *Ops[] = {Acc, Src};
176 Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
177 Src->getType()};
178 auto Decl = Intrinsic::getDeclaration(
179 M, Intrinsic::experimental_vector_reduce_fadd, Tys);
180 return createCallHelper(Decl, Ops, this);
181}
182
183CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
184 Module *M = GetInsertBlock()->getParent()->getParent();
185 Value *Ops[] = {Acc, Src};
186 Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
187 Src->getType()};
188 auto Decl = Intrinsic::getDeclaration(
189 M, Intrinsic::experimental_vector_reduce_fmul, Tys);
190 return createCallHelper(Decl, Ops, this);
191}
192
193CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
194 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
195 Src);
196}
197
198CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
199 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
200 Src);
201}
202
203CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
204 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
205 Src);
206}
207
208CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
209 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
210 Src);
211}
212
213CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
214 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
215 Src);
216}
217
218CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
219 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
220 : Intrinsic::experimental_vector_reduce_umax;
221 return getReductionIntrinsic(this, ID, Src);
222}
223
224CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
225 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
226 : Intrinsic::experimental_vector_reduce_umin;
227 return getReductionIntrinsic(this, ID, Src);
228}
229
230CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
231 auto Rdx = getReductionIntrinsic(
232 this, Intrinsic::experimental_vector_reduce_fmax, Src);
233 if (NoNaN) {
234 FastMathFlags FMF;
235 FMF.setNoNaNs();
236 Rdx->setFastMathFlags(FMF);
237 }
238 return Rdx;
239}
240
241CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
242 auto Rdx = getReductionIntrinsic(
243 this, Intrinsic::experimental_vector_reduce_fmin, Src);
244 if (NoNaN) {
245 FastMathFlags FMF;
246 FMF.setNoNaNs();
247 Rdx->setFastMathFlags(FMF);
248 }
249 return Rdx;
250}
251
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000252CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
253 assert(isa<PointerType>(Ptr->getType()) &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000254 "lifetime.start only applies to pointers.");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000255 Ptr = getCastedInt8PtrValue(Ptr);
256 if (!Size)
257 Size = getInt64(-1);
258 else
259 assert(Size->getType() == getInt64Ty() &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000260 "lifetime.start requires the size to be an i64");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000261 Value *Ops[] = { Size, Ptr };
262 Module *M = BB->getParent()->getParent();
Matt Arsenaultf10061e2017-04-10 20:18:21 +0000263 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_start,
264 { Ptr->getType() });
Jay Foad5bd375a2011-07-15 08:37:34 +0000265 return createCallHelper(TheFn, Ops, this);
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000266}
267
268CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
269 assert(isa<PointerType>(Ptr->getType()) &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000270 "lifetime.end only applies to pointers.");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000271 Ptr = getCastedInt8PtrValue(Ptr);
272 if (!Size)
273 Size = getInt64(-1);
274 else
275 assert(Size->getType() == getInt64Ty() &&
Bill Wendlingea6397f2012-07-19 00:11:40 +0000276 "lifetime.end requires the size to be an i64");
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000277 Value *Ops[] = { Size, Ptr };
278 Module *M = BB->getParent()->getParent();
Matt Arsenaultf10061e2017-04-10 20:18:21 +0000279 Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_end,
280 { Ptr->getType() });
Jay Foad5bd375a2011-07-15 08:37:34 +0000281 return createCallHelper(TheFn, Ops, this);
Nick Lewyckybabca9a2011-05-21 23:14:36 +0000282}
Hal Finkel6f814db2014-10-15 23:44:22 +0000283
Anna Thomas58d11922016-07-22 20:57:23 +0000284CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
285
286 assert(isa<PointerType>(Ptr->getType()) &&
287 "invariant.start only applies to pointers.");
288 Ptr = getCastedInt8PtrValue(Ptr);
289 if (!Size)
290 Size = getInt64(-1);
291 else
292 assert(Size->getType() == getInt64Ty() &&
293 "invariant.start requires the size to be an i64");
294
295 Value *Ops[] = {Size, Ptr};
296 // Fill in the single overloaded type: memory object type.
297 Type *ObjectPtr[1] = {Ptr->getType()};
298 Module *M = BB->getParent()->getParent();
299 Value *TheFn =
300 Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
301 return createCallHelper(TheFn, Ops, this);
302}
303
Hal Finkel6f814db2014-10-15 23:44:22 +0000304CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
305 assert(Cond->getType() == getInt1Ty() &&
306 "an assumption condition must be of type i1");
307
308 Value *Ops[] = { Cond };
309 Module *M = BB->getParent()->getParent();
310 Value *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
311 return createCallHelper(FnAssume, Ops, this);
312}
313
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000314/// \brief Create a call to a Masked Load intrinsic.
315/// \p Ptr - base pointer for the load
316/// \p Align - alignment of the source location
317/// \p Mask - vector of booleans which indicates what vector lanes should
318/// be accessed in memory
319/// \p PassThru - pass-through value that is used to fill the masked-off lanes
320/// of the result
321/// \p Name - name of the result variable
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000322CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
323 Value *Mask, Value *PassThru,
324 const Twine &Name) {
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000325 PointerType *PtrTy = cast<PointerType>(Ptr->getType());
326 Type *DataTy = PtrTy->getElementType();
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000327 assert(DataTy->isVectorTy() && "Ptr should point to a vector");
328 if (!PassThru)
329 PassThru = UndefValue::get(DataTy);
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000330 Type *OverloadedTypes[] = { DataTy, PtrTy };
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000331 Value *Ops[] = { Ptr, getInt32(Align), Mask, PassThru};
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000332 return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
333 OverloadedTypes, Name);
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000334}
335
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000336/// \brief Create a call to a Masked Store intrinsic.
337/// \p Val - data to be stored,
338/// \p Ptr - base pointer for the store
339/// \p Align - alignment of the destination location
340/// \p Mask - vector of booleans which indicates what vector lanes should
341/// be accessed in memory
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000342CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
343 unsigned Align, Value *Mask) {
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000344 PointerType *PtrTy = cast<PointerType>(Ptr->getType());
345 Type *DataTy = PtrTy->getElementType();
346 assert(DataTy->isVectorTy() && "Ptr should point to a vector");
347 Type *OverloadedTypes[] = { DataTy, PtrTy };
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000348 Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000349 return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000350}
351
352/// Create a call to a Masked intrinsic, with given intrinsic Id,
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000353/// an array of operands - Ops, and an array of overloaded types -
354/// OverloadedTypes.
Pete Cooper9e1d3352015-05-20 17:16:39 +0000355CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000356 ArrayRef<Value *> Ops,
Artur Pilipenko7ad95ec2016-06-28 18:27:25 +0000357 ArrayRef<Type *> OverloadedTypes,
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000358 const Twine &Name) {
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000359 Module *M = BB->getParent()->getParent();
Pete Cooper9e1d3352015-05-20 17:16:39 +0000360 Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
Elena Demikhovsky84d19972014-12-30 14:28:14 +0000361 return createCallHelper(TheFn, Ops, this, Name);
Elena Demikhovskyf1de34b2014-12-04 09:40:44 +0000362}
Philip Reames4750dd12014-12-30 05:55:58 +0000363
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000364/// \brief Create a call to a Masked Gather intrinsic.
365/// \p Ptrs - vector of pointers for loading
366/// \p Align - alignment for one element
367/// \p Mask - vector of booleans which indicates what vector lanes should
368/// be accessed in memory
369/// \p PassThru - pass-through value that is used to fill the masked-off lanes
370/// of the result
371/// \p Name - name of the result variable
372CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
373 Value *Mask, Value *PassThru,
374 const Twine& Name) {
375 auto PtrsTy = cast<VectorType>(Ptrs->getType());
376 auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
377 unsigned NumElts = PtrsTy->getVectorNumElements();
378 Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
379
380 if (!Mask)
381 Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
382 NumElts));
383
Amara Emerson4d33c862017-05-19 10:40:18 +0000384 if (!PassThru)
385 PassThru = UndefValue::get(DataTy);
386
Elad Cohenef5798a2017-05-03 12:28:54 +0000387 Type *OverloadedTypes[] = {DataTy, PtrsTy};
Amara Emerson4d33c862017-05-19 10:40:18 +0000388 Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000389
390 // We specify only one type when we create this intrinsic. Types of other
391 // arguments are derived from this type.
Elad Cohenef5798a2017-05-03 12:28:54 +0000392 return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
393 Name);
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000394}
395
396/// \brief Create a call to a Masked Scatter intrinsic.
397/// \p Data - data to be stored,
398/// \p Ptrs - the vector of pointers, where the \p Data elements should be
399/// stored
400/// \p Align - alignment for one element
401/// \p Mask - vector of booleans which indicates what vector lanes should
402/// be accessed in memory
403CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
404 unsigned Align, Value *Mask) {
405 auto PtrsTy = cast<VectorType>(Ptrs->getType());
406 auto DataTy = cast<VectorType>(Data->getType());
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000407 unsigned NumElts = PtrsTy->getVectorNumElements();
408
Tim Northover5a1a56c2016-02-17 21:16:59 +0000409#ifndef NDEBUG
410 auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000411 assert(NumElts == DataTy->getVectorNumElements() &&
Tim Northover5a1a56c2016-02-17 21:16:59 +0000412 PtrTy->getElementType() == DataTy->getElementType() &&
413 "Incompatible pointer and data types");
414#endif
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000415
416 if (!Mask)
417 Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
418 NumElts));
Elad Cohenef5798a2017-05-03 12:28:54 +0000419
420 Type *OverloadedTypes[] = {DataTy, PtrsTy};
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000421 Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
422
423 // We specify only one type when we create this intrinsic. Types of other
424 // arguments are derived from this type.
Elad Cohenef5798a2017-05-03 12:28:54 +0000425 return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
Elena Demikhovsky88e76ca2016-02-17 19:23:04 +0000426}
427
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000428template <typename T0, typename T1, typename T2, typename T3>
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000429static std::vector<Value *>
430getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000431 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
432 ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
433 ArrayRef<T3> GCArgs) {
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000434 std::vector<Value *> Args;
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000435 Args.push_back(B.getInt64(ID));
436 Args.push_back(B.getInt32(NumPatchBytes));
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000437 Args.push_back(ActualCallee);
438 Args.push_back(B.getInt32(CallArgs.size()));
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000439 Args.push_back(B.getInt32(Flags));
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000440 Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000441 Args.push_back(B.getInt32(TransitionArgs.size()));
442 Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000443 Args.push_back(B.getInt32(DeoptArgs.size()));
444 Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
445 Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
446
447 return Args;
448}
449
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000450template <typename T0, typename T1, typename T2, typename T3>
451static CallInst *CreateGCStatepointCallCommon(
452 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000453 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000454 ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
455 const Twine &Name) {
Sanjoy Das63245b52015-05-06 02:36:34 +0000456 // Extract out the type of the callee.
457 PointerType *FuncPtrType = cast<PointerType>(ActualCallee->getType());
458 assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
459 "actual callee must be a callable value");
Philip Reames4750dd12014-12-30 05:55:58 +0000460
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000461 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
Sanjoy Das63245b52015-05-06 02:36:34 +0000462 // Fill in the one generic type'd argument (the function is also vararg)
463 Type *ArgTypes[] = { FuncPtrType };
464 Function *FnStatepoint =
465 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
466 ArgTypes);
Philip Reames4750dd12014-12-30 05:55:58 +0000467
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000468 std::vector<llvm::Value *> Args =
469 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
470 CallArgs, TransitionArgs, DeoptArgs, GCArgs);
471 return createCallHelper(FnStatepoint, Args, Builder, Name);
472}
473
474CallInst *IRBuilderBase::CreateGCStatepointCall(
475 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
476 ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
477 ArrayRef<Value *> GCArgs, const Twine &Name) {
478 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000479 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
480 CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000481}
482
483CallInst *IRBuilderBase::CreateGCStatepointCall(
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000484 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
485 ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000486 ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
487 return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
488 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
489 DeoptArgs, GCArgs, Name);
Philip Reames4750dd12014-12-30 05:55:58 +0000490}
491
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000492CallInst *IRBuilderBase::CreateGCStatepointCall(
493 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
494 ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
495 ArrayRef<Value *> GCArgs, const Twine &Name) {
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000496 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000497 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
498 CallArgs, None, DeoptArgs, GCArgs, Name);
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000499}
500
501template <typename T0, typename T1, typename T2, typename T3>
502static InvokeInst *CreateGCStatepointInvokeCommon(
503 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
504 Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000505 uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000506 ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
507 // Extract out the type of the callee.
508 PointerType *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
509 assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
510 "actual callee must be a callable value");
511
512 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
513 // Fill in the one generic type'd argument (the function is also vararg)
514 Function *FnStatepoint = Intrinsic::getDeclaration(
515 M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
516
517 std::vector<llvm::Value *> Args =
518 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
519 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
520 return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
521 Name);
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000522}
523
524InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000525 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
526 BasicBlock *NormalDest, BasicBlock *UnwindDest,
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000527 ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
528 ArrayRef<Value *> GCArgs, const Twine &Name) {
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000529 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
530 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000531 uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000532 DeoptArgs, GCArgs, Name);
533}
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000534
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000535InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
536 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000537 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000538 ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
539 ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
540 return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
541 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
542 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
Sanjoy Dasabe1c682015-05-06 23:53:09 +0000543}
544
545InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
Sanjoy Dasa1d39ba2015-05-12 23:52:24 +0000546 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
547 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
548 ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
Sanjoy Dasaf6980c2015-10-07 19:52:12 +0000549 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
550 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
Sanjoy Das4fd3d402015-10-08 23:18:33 +0000551 uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
552 Name);
Ramkumar Ramachandra3408f3e2015-02-26 00:35:56 +0000553}
554
Philip Reames4750dd12014-12-30 05:55:58 +0000555CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
556 Type *ResultType,
557 const Twine &Name) {
Ramkumar Ramachandra75a4f352015-01-22 20:14:38 +0000558 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
Philip Reames4750dd12014-12-30 05:55:58 +0000559 Module *M = BB->getParent()->getParent();
560 Type *Types[] = {ResultType};
561 Value *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
562
563 Value *Args[] = {Statepoint};
564 return createCallHelper(FnGCResult, Args, this, Name);
565}
566
567CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
568 int BaseOffset,
569 int DerivedOffset,
570 Type *ResultType,
571 const Twine &Name) {
572 Module *M = BB->getParent()->getParent();
573 Type *Types[] = {ResultType};
574 Value *FnGCRelocate =
575 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
576
577 Value *Args[] = {Statepoint,
578 getInt32(BaseOffset),
579 getInt32(DerivedOffset)};
580 return createCallHelper(FnGCRelocate, Args, this, Name);
581}
Matt Arsenaultcdb468c2017-02-27 23:08:49 +0000582
583CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID,
584 Value *LHS, Value *RHS,
585 const Twine &Name) {
586 Module *M = BB->getParent()->getParent();
587 Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
588 return createCallHelper(Fn, { LHS, RHS }, this, Name);
589}