blob: 72847fa2827db6909c46882650da380602adef95 [file] [log] [blame]
Alexey Bataev9959db52014-05-06 10:08:46 +00001//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This contains code to emit OpenMP nodes as LLVM code.
11//
12//===----------------------------------------------------------------------===//
13
Alexey Bataev3392d762016-02-16 11:18:12 +000014#include "CGCleanup.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000015#include "CGOpenMPRuntime.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
Chandler Carruth0d9593d2015-01-14 11:29:14 +000018#include "TargetInfo.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000019#include "clang/AST/Stmt.h"
20#include "clang/AST/StmtOpenMP.h"
Alexey Bataev2bbf7212016-03-03 03:52:24 +000021#include "clang/AST/DeclOpenMP.h"
Alexey Bataeva839ddd2016-03-17 10:19:46 +000022#include "llvm/IR/CallSite.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000023using namespace clang;
24using namespace CodeGen;
25
Alexey Bataev3392d762016-02-16 11:18:12 +000026namespace {
27/// Lexical scope for OpenMP executable constructs, that handles correct codegen
28/// for captured expressions.
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000029class OMPLexicalScope : public CodeGenFunction::LexicalScope {
Alexey Bataev3392d762016-02-16 11:18:12 +000030 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31 for (const auto *C : S.clauses()) {
32 if (auto *CPI = OMPClauseWithPreInit::get(C)) {
33 if (auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
Alexey Bataev2bbf7212016-03-03 03:52:24 +000034 for (const auto *I : PreInit->decls()) {
35 if (!I->hasAttr<OMPCaptureNoInitAttr>())
36 CGF.EmitVarDecl(cast<VarDecl>(*I));
37 else {
38 CodeGenFunction::AutoVarEmission Emission =
39 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
40 CGF.EmitAutoVarCleanups(Emission);
41 }
42 }
Alexey Bataev3392d762016-02-16 11:18:12 +000043 }
44 }
45 }
46 }
Alexey Bataev4ba78a42016-04-27 07:56:03 +000047 CodeGenFunction::OMPPrivateScope InlinedShareds;
48
49 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
50 return CGF.LambdaCaptureFields.lookup(VD) ||
51 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
52 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
53 }
Alexey Bataev3392d762016-02-16 11:18:12 +000054
Alexey Bataev3392d762016-02-16 11:18:12 +000055public:
Alexey Bataev4ba78a42016-04-27 07:56:03 +000056 OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S,
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000057 bool AsInlined = false, bool EmitPreInitStmt = true)
Alexey Bataev4ba78a42016-04-27 07:56:03 +000058 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
59 InlinedShareds(CGF) {
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000060 if (EmitPreInitStmt)
61 emitPreInitStmt(CGF, S);
Alexey Bataev4ba78a42016-04-27 07:56:03 +000062 if (AsInlined) {
63 if (S.hasAssociatedStmt()) {
64 auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
65 for (auto &C : CS->captures()) {
66 if (C.capturesVariable() || C.capturesVariableByCopy()) {
67 auto *VD = C.getCapturedVar();
Alexey Bataev6a71f362017-08-22 17:54:52 +000068 assert(VD == VD->getCanonicalDecl() &&
69 "Canonical decl must be captured.");
Alexey Bataev4ba78a42016-04-27 07:56:03 +000070 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
71 isCapturedVar(CGF, VD) ||
72 (CGF.CapturedStmtInfo &&
73 InlinedShareds.isGlobalVarCaptured(VD)),
74 VD->getType().getNonReferenceType(), VK_LValue,
75 SourceLocation());
76 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
77 return CGF.EmitLValue(&DRE).getAddress();
78 });
79 }
80 }
81 (void)InlinedShareds.Privatize();
82 }
83 }
Alexey Bataev3392d762016-02-16 11:18:12 +000084 }
85};
Alexey Bataev14fa1c62016-03-29 05:34:15 +000086
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000087/// Lexical scope for OpenMP parallel construct, that handles correct codegen
88/// for captured expressions.
89class OMPParallelScope final : public OMPLexicalScope {
90 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
91 OpenMPDirectiveKind Kind = S.getDirectiveKind();
Carlo Bertollib0ff0a62017-04-25 17:52:12 +000092 return !(isOpenMPTargetExecutionDirective(Kind) ||
93 isOpenMPLoopBoundSharingDirective(Kind)) &&
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000094 isOpenMPParallelDirective(Kind);
95 }
96
97public:
98 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99 : OMPLexicalScope(CGF, S,
100 /*AsInlined=*/false,
101 /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
102};
103
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +0000104/// Lexical scope for OpenMP teams construct, that handles correct codegen
105/// for captured expressions.
106class OMPTeamsScope final : public OMPLexicalScope {
107 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
108 OpenMPDirectiveKind Kind = S.getDirectiveKind();
109 return !isOpenMPTargetExecutionDirective(Kind) &&
110 isOpenMPTeamsDirective(Kind);
111 }
112
113public:
114 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
115 : OMPLexicalScope(CGF, S,
116 /*AsInlined=*/false,
117 /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
118};
119
Alexey Bataev5a3af132016-03-29 08:58:54 +0000120/// Private scope for OpenMP loop-based directives, that supports capturing
121/// of used expression from loop statement.
122class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124 if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
125 if (auto *PreInits = cast_or_null<DeclStmt>(LD->getPreInits())) {
126 for (const auto *I : PreInits->decls())
127 CGF.EmitVarDecl(cast<VarDecl>(*I));
128 }
129 }
130 }
131
132public:
133 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
134 : CodeGenFunction::RunCleanupsScope(CGF) {
135 emitPreInitStmt(CGF, S);
136 }
137};
138
Alexey Bataev3392d762016-02-16 11:18:12 +0000139} // namespace
140
Alexey Bataevf8365372017-11-17 17:57:25 +0000141static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
142 const OMPExecutableDirective &S,
143 const RegionCodeGenTy &CodeGen);
144
Alexey Bataevf47c4b42017-09-26 13:47:31 +0000145LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
146 if (auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
147 if (auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
148 OrigVD = OrigVD->getCanonicalDecl();
149 bool IsCaptured =
150 LambdaCaptureFields.lookup(OrigVD) ||
151 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
152 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
153 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
154 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
155 return EmitLValue(&DRE);
156 }
157 }
158 return EmitLValue(E);
159}
160
Alexey Bataev1189bd02016-01-26 12:20:39 +0000161llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
162 auto &C = getContext();
163 llvm::Value *Size = nullptr;
164 auto SizeInChars = C.getTypeSizeInChars(Ty);
165 if (SizeInChars.isZero()) {
166 // getTypeSizeInChars() returns 0 for a VLA.
167 while (auto *VAT = C.getAsVariableArrayType(Ty)) {
168 llvm::Value *ArraySize;
169 std::tie(ArraySize, Ty) = getVLASize(VAT);
170 Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
171 }
172 SizeInChars = C.getTypeSizeInChars(Ty);
173 if (SizeInChars.isZero())
174 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
175 Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
176 } else
177 Size = CGM.getSize(SizeInChars);
178 return Size;
179}
180
Alexey Bataev2377fe92015-09-10 08:12:02 +0000181void CodeGenFunction::GenerateOpenMPCapturedVars(
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000182 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
Alexey Bataev2377fe92015-09-10 08:12:02 +0000183 const RecordDecl *RD = S.getCapturedRecordDecl();
184 auto CurField = RD->field_begin();
185 auto CurCap = S.captures().begin();
186 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
187 E = S.capture_init_end();
188 I != E; ++I, ++CurField, ++CurCap) {
189 if (CurField->hasCapturedVLAType()) {
190 auto VAT = CurField->getCapturedVLAType();
Samuel Antaobed3c462015-10-02 16:14:20 +0000191 auto *Val = VLASizeMap[VAT->getSizeExpr()];
Samuel Antaobed3c462015-10-02 16:14:20 +0000192 CapturedVars.push_back(Val);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000193 } else if (CurCap->capturesThis())
194 CapturedVars.push_back(CXXThisValue);
Samuel Antao6d004262016-06-16 18:39:34 +0000195 else if (CurCap->capturesVariableByCopy()) {
196 llvm::Value *CV =
197 EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal();
198
199 // If the field is not a pointer, we need to save the actual value
200 // and load it as a void pointer.
201 if (!CurField->getType()->isAnyPointerType()) {
202 auto &Ctx = getContext();
203 auto DstAddr = CreateMemTemp(
204 Ctx.getUIntPtrType(),
205 Twine(CurCap->getCapturedVar()->getName()) + ".casted");
206 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
207
208 auto *SrcAddrVal = EmitScalarConversion(
209 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
210 Ctx.getPointerType(CurField->getType()), SourceLocation());
211 LValue SrcLV =
212 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
213
214 // Store the value using the source type pointer.
215 EmitStoreThroughLValue(RValue::get(CV), SrcLV);
216
217 // Load the value using the destination type pointer.
218 CV = EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
219 }
220 CapturedVars.push_back(CV);
221 } else {
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000222 assert(CurCap->capturesVariable() && "Expected capture by reference.");
Alexey Bataev2377fe92015-09-10 08:12:02 +0000223 CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000224 }
Alexey Bataev2377fe92015-09-10 08:12:02 +0000225 }
226}
227
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000228static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType,
229 StringRef Name, LValue AddrLV,
230 bool isReferenceType = false) {
231 ASTContext &Ctx = CGF.getContext();
232
233 auto *CastedPtr = CGF.EmitScalarConversion(
234 AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
235 Ctx.getPointerType(DstType), SourceLocation());
236 auto TmpAddr =
237 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
238 .getAddress();
239
240 // If we are dealing with references we need to return the address of the
241 // reference instead of the reference of the value.
242 if (isReferenceType) {
243 QualType RefType = Ctx.getLValueReferenceType(DstType);
244 auto *RefVal = TmpAddr.getPointer();
245 TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
246 auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
Akira Hatanaka642f7992016-10-18 19:05:41 +0000247 CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true);
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000248 }
249
250 return TmpAddr;
251}
252
Alexey Bataevf7ce1662017-04-10 19:16:45 +0000253static QualType getCanonicalParamType(ASTContext &C, QualType T) {
254 if (T->isLValueReferenceType()) {
255 return C.getLValueReferenceType(
256 getCanonicalParamType(C, T.getNonReferenceType()),
257 /*SpelledAsLValue=*/false);
258 }
259 if (T->isPointerType())
260 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
Alexey Bataev1b48c5e2017-10-24 19:52:31 +0000261 if (auto *A = T->getAsArrayTypeUnsafe()) {
262 if (auto *VLA = dyn_cast<VariableArrayType>(A))
263 return getCanonicalParamType(C, VLA->getElementType());
264 else if (!A->isVariablyModifiedType())
265 return C.getCanonicalType(T);
266 }
Alexey Bataevf7ce1662017-04-10 19:16:45 +0000267 return C.getCanonicalParamType(T);
268}
269
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000270namespace {
271 /// Contains required data for proper outlined function codegen.
272 struct FunctionOptions {
273 /// Captured statement for which the function is generated.
274 const CapturedStmt *S = nullptr;
275 /// true if cast to/from UIntPtr is required for variables captured by
276 /// value.
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000277 const bool UIntPtrCastRequired = true;
Alexey Bataeve754b182017-08-09 19:38:53 +0000278 /// true if only casted arguments must be registered as local args or VLA
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000279 /// sizes.
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000280 const bool RegisterCastedArgsOnly = false;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000281 /// Name of the generated function.
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000282 const StringRef FunctionName;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000283 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
284 bool RegisterCastedArgsOnly,
Alexey Bataev4aa19052017-08-08 16:45:36 +0000285 StringRef FunctionName)
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000286 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
287 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
Alexey Bataev4aa19052017-08-08 16:45:36 +0000288 FunctionName(FunctionName) {}
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000289 };
290}
291
Alexey Bataeve754b182017-08-09 19:38:53 +0000292static llvm::Function *emitOutlinedFunctionPrologue(
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000293 CodeGenFunction &CGF, FunctionArgList &Args,
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000294 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000295 &LocalAddrs,
296 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
297 &VLASizes,
298 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
299 const CapturedDecl *CD = FO.S->getCapturedDecl();
300 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000301 assert(CD->hasBody() && "missing CapturedDecl body");
302
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000303 CXXThisValue = nullptr;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000304 // Build the argument list.
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000305 CodeGenModule &CGM = CGF.CGM;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000306 ASTContext &Ctx = CGM.getContext();
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000307 FunctionArgList TargetArgs;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000308 Args.append(CD->param_begin(),
309 std::next(CD->param_begin(), CD->getContextParamPosition()));
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000310 TargetArgs.append(
311 CD->param_begin(),
312 std::next(CD->param_begin(), CD->getContextParamPosition()));
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000313 auto I = FO.S->captures().begin();
Alexey Bataevb45d43c2017-11-22 16:02:03 +0000314 FunctionDecl *DebugFunctionDecl = nullptr;
315 if (!FO.UIntPtrCastRequired) {
316 FunctionProtoType::ExtProtoInfo EPI;
317 DebugFunctionDecl = FunctionDecl::Create(
318 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(),
319 SourceLocation(), DeclarationName(), Ctx.VoidTy,
320 Ctx.getTrivialTypeSourceInfo(
321 Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
322 SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
323 }
Alexey Bataev2377fe92015-09-10 08:12:02 +0000324 for (auto *FD : RD->fields()) {
325 QualType ArgType = FD->getType();
326 IdentifierInfo *II = nullptr;
327 VarDecl *CapVar = nullptr;
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000328
329 // If this is a capture by copy and the type is not a pointer, the outlined
330 // function argument type should be uintptr and the value properly casted to
331 // uintptr. This is necessary given that the runtime library is only able to
332 // deal with pointers. We can pass in the same way the VLA type sizes to the
333 // outlined function.
Samuel Antao6d004262016-06-16 18:39:34 +0000334 if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000335 I->capturesVariableArrayType()) {
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000336 if (FO.UIntPtrCastRequired)
337 ArgType = Ctx.getUIntPtrType();
338 }
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000339
340 if (I->capturesVariable() || I->capturesVariableByCopy()) {
Alexey Bataev2377fe92015-09-10 08:12:02 +0000341 CapVar = I->getCapturedVar();
342 II = CapVar->getIdentifier();
343 } else if (I->capturesThis())
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000344 II = &Ctx.Idents.get("this");
Alexey Bataev2377fe92015-09-10 08:12:02 +0000345 else {
346 assert(I->capturesVariableArrayType());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000347 II = &Ctx.Idents.get("vla");
Alexey Bataev2377fe92015-09-10 08:12:02 +0000348 }
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000349 if (ArgType->isVariablyModifiedType())
Alexey Bataev1b48c5e2017-10-24 19:52:31 +0000350 ArgType = getCanonicalParamType(Ctx, ArgType);
Alexey Bataevb45d43c2017-11-22 16:02:03 +0000351 VarDecl *Arg;
352 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
353 Arg = ParmVarDecl::Create(
354 Ctx, DebugFunctionDecl,
355 CapVar ? CapVar->getLocStart() : FD->getLocStart(),
356 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
357 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
358 } else {
359 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
360 II, ArgType, ImplicitParamDecl::Other);
361 }
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000362 Args.emplace_back(Arg);
363 // Do not cast arguments if we emit function with non-original types.
364 TargetArgs.emplace_back(
365 FO.UIntPtrCastRequired
366 ? Arg
367 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
Alexey Bataev2377fe92015-09-10 08:12:02 +0000368 ++I;
369 }
370 Args.append(
371 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
372 CD->param_end());
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000373 TargetArgs.append(
374 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
375 CD->param_end());
Alexey Bataev2377fe92015-09-10 08:12:02 +0000376
377 // Create the function declaration.
Alexey Bataev2377fe92015-09-10 08:12:02 +0000378 const CGFunctionInfo &FuncInfo =
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000379 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000380 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
381
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000382 llvm::Function *F =
383 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
384 FO.FunctionName, &CGM.getModule());
Alexey Bataev2377fe92015-09-10 08:12:02 +0000385 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
386 if (CD->isNothrow())
Alexey Bataev2c7eee52017-08-04 19:10:54 +0000387 F->setDoesNotThrow();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000388
389 // Generate the function.
Alexey Bataev6e01dc12017-08-14 16:03:47 +0000390 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
391 FO.S->getLocStart(), CD->getBody()->getLocStart());
Alexey Bataev2377fe92015-09-10 08:12:02 +0000392 unsigned Cnt = CD->getContextParamPosition();
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000393 I = FO.S->captures().begin();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000394 for (auto *FD : RD->fields()) {
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000395 // Do not map arguments if we emit function with non-original types.
396 Address LocalAddr(Address::invalid());
397 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
398 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
399 TargetArgs[Cnt]);
400 } else {
401 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
402 }
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000403 // If we are capturing a pointer by copy we don't need to do anything, just
404 // use the value that we get from the arguments.
405 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
Samuel Antao403ffd42016-07-27 22:49:49 +0000406 const VarDecl *CurVD = I->getCapturedVar();
Samuel Antao403ffd42016-07-27 22:49:49 +0000407 // If the variable is a reference we need to materialize it here.
408 if (CurVD->getType()->isReferenceType()) {
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000409 Address RefAddr = CGF.CreateMemTemp(
410 CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
411 CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
412 /*Volatile=*/false, CurVD->getType());
Samuel Antao403ffd42016-07-27 22:49:49 +0000413 LocalAddr = RefAddr;
414 }
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000415 if (!FO.RegisterCastedArgsOnly)
416 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
Richard Trieucc3949d2016-02-18 22:34:54 +0000417 ++Cnt;
418 ++I;
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000419 continue;
420 }
421
Ivan A. Kosarev5f8c0ca2017-10-10 09:39:32 +0000422 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
423 AlignmentSource::Decl);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000424 if (FD->hasCapturedVLAType()) {
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000425 if (FO.UIntPtrCastRequired) {
426 ArgLVal = CGF.MakeAddrLValue(castValueFromUintptr(CGF, FD->getType(),
427 Args[Cnt]->getName(),
428 ArgLVal),
Ivan A. Kosarev5f8c0ca2017-10-10 09:39:32 +0000429 FD->getType(), AlignmentSource::Decl);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000430 }
Alexey Bataev2377fe92015-09-10 08:12:02 +0000431 auto *ExprArg =
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000432 CGF.EmitLoadOfLValue(ArgLVal, SourceLocation()).getScalarVal();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000433 auto VAT = FD->getCapturedVLAType();
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000434 VLASizes.insert({Args[Cnt], {VAT->getSizeExpr(), ExprArg}});
Alexey Bataev2377fe92015-09-10 08:12:02 +0000435 } else if (I->capturesVariable()) {
436 auto *Var = I->getCapturedVar();
437 QualType VarTy = Var->getType();
438 Address ArgAddr = ArgLVal.getAddress();
439 if (!VarTy->isReferenceType()) {
Alexey Bataev2f5ed342016-10-13 09:52:46 +0000440 if (ArgLVal.getType()->isLValueReferenceType()) {
Ivan A. Kosarev9f9d1572017-10-30 11:49:31 +0000441 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
Alexey Bataevac5eabb2016-11-07 11:16:04 +0000442 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
Alexey Bataev2f5ed342016-10-13 09:52:46 +0000443 assert(ArgLVal.getType()->isPointerType());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000444 ArgAddr = CGF.EmitLoadOfPointer(
Alexey Bataev2f5ed342016-10-13 09:52:46 +0000445 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
446 }
Alexey Bataev2377fe92015-09-10 08:12:02 +0000447 }
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000448 if (!FO.RegisterCastedArgsOnly) {
449 LocalAddrs.insert(
450 {Args[Cnt],
451 {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
452 }
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000453 } else if (I->capturesVariableByCopy()) {
454 assert(!FD->getType()->isAnyPointerType() &&
455 "Not expecting a captured pointer.");
456 auto *Var = I->getCapturedVar();
457 QualType VarTy = Var->getType();
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000458 LocalAddrs.insert(
459 {Args[Cnt],
460 {Var,
461 FO.UIntPtrCastRequired
462 ? castValueFromUintptr(CGF, FD->getType(), Args[Cnt]->getName(),
463 ArgLVal, VarTy->isReferenceType())
464 : ArgLVal.getAddress()}});
Alexey Bataev2377fe92015-09-10 08:12:02 +0000465 } else {
466 // If 'this' is captured, load it into CXXThisValue.
467 assert(I->capturesThis());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000468 CXXThisValue = CGF.EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation())
469 .getScalarVal();
470 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
Alexey Bataev2377fe92015-09-10 08:12:02 +0000471 }
Richard Trieucc3949d2016-02-18 22:34:54 +0000472 ++Cnt;
473 ++I;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000474 }
475
Alexey Bataeve754b182017-08-09 19:38:53 +0000476 return F;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000477}
478
479llvm::Function *
480CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
481 assert(
482 CapturedStmtInfo &&
483 "CapturedStmtInfo should be set when generating the captured function");
484 const CapturedDecl *CD = S.getCapturedDecl();
485 // Build the argument list.
486 bool NeedWrapperFunction =
487 getDebugInfo() &&
488 CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
489 FunctionArgList Args;
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000490 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000491 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
Alexey Bataeve754b182017-08-09 19:38:53 +0000492 SmallString<256> Buffer;
493 llvm::raw_svector_ostream Out(Buffer);
494 Out << CapturedStmtInfo->getHelperName();
495 if (NeedWrapperFunction)
496 Out << "_debug__";
Alexey Bataev4aa19052017-08-08 16:45:36 +0000497 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
Alexey Bataeve754b182017-08-09 19:38:53 +0000498 Out.str());
499 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
500 VLASizes, CXXThisValue, FO);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000501 for (const auto &LocalAddrPair : LocalAddrs) {
502 if (LocalAddrPair.second.first) {
503 setAddrOfLocalVar(LocalAddrPair.second.first,
504 LocalAddrPair.second.second);
505 }
506 }
507 for (const auto &VLASizePair : VLASizes)
508 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
Serge Pavlov3a561452015-12-06 14:32:39 +0000509 PGO.assignRegionCounters(GlobalDecl(CD), F);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000510 CapturedStmtInfo->EmitBody(*this, CD->getBody());
511 FinishFunction(CD->getBodyRBrace());
Alexey Bataeve754b182017-08-09 19:38:53 +0000512 if (!NeedWrapperFunction)
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000513 return F;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000514
Alexey Bataevefd884d2017-08-04 21:26:25 +0000515 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
Alexey Bataeve754b182017-08-09 19:38:53 +0000516 /*RegisterCastedArgsOnly=*/true,
517 CapturedStmtInfo->getHelperName());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000518 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000519 Args.clear();
520 LocalAddrs.clear();
521 VLASizes.clear();
522 llvm::Function *WrapperF =
523 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
Alexey Bataeve754b182017-08-09 19:38:53 +0000524 WrapperCGF.CXXThisValue, WrapperFO);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000525 llvm::SmallVector<llvm::Value *, 4> CallArgs;
526 for (const auto *Arg : Args) {
527 llvm::Value *CallArg;
528 auto I = LocalAddrs.find(Arg);
529 if (I != LocalAddrs.end()) {
Alexey Bataev7ba57af2017-10-17 16:47:34 +0000530 LValue LV = WrapperCGF.MakeAddrLValue(
531 I->second.second,
532 I->second.first ? I->second.first->getType() : Arg->getType(),
533 AlignmentSource::Decl);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000534 CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
535 } else {
536 auto EI = VLASizes.find(Arg);
537 if (EI != VLASizes.end())
538 CallArg = EI->second.second;
539 else {
540 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
Ivan A. Kosarev5f8c0ca2017-10-10 09:39:32 +0000541 Arg->getType(),
542 AlignmentSource::Decl);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000543 CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
544 }
545 }
Alexey Bataev7ba57af2017-10-17 16:47:34 +0000546 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000547 }
Alexey Bataev3c595a62017-08-14 15:01:03 +0000548 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(),
549 F, CallArgs);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000550 WrapperCGF.FinishFunction();
551 return WrapperF;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000552}
553
Alexey Bataev9959db52014-05-06 10:08:46 +0000554//===----------------------------------------------------------------------===//
555// OpenMP Directive Emission
556//===----------------------------------------------------------------------===//
Alexey Bataev420d45b2015-04-14 05:11:24 +0000557void CodeGenFunction::EmitOMPAggregateAssign(
John McCall7f416cc2015-09-08 08:05:57 +0000558 Address DestAddr, Address SrcAddr, QualType OriginalType,
559 const llvm::function_ref<void(Address, Address)> &CopyGen) {
Alexey Bataev420d45b2015-04-14 05:11:24 +0000560 // Perform element-by-element initialization.
561 QualType ElementTy;
John McCall7f416cc2015-09-08 08:05:57 +0000562
563 // Drill down to the base element type on both arrays.
Alexey Bataev420d45b2015-04-14 05:11:24 +0000564 auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
John McCall7f416cc2015-09-08 08:05:57 +0000565 auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
566 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
567
568 auto SrcBegin = SrcAddr.getPointer();
569 auto DestBegin = DestAddr.getPointer();
Alexey Bataev420d45b2015-04-14 05:11:24 +0000570 // Cast from pointer to array type to pointer to single element.
Alexey Bataev420d45b2015-04-14 05:11:24 +0000571 auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
572 // The basic structure here is a while-do loop.
573 auto BodyBB = createBasicBlock("omp.arraycpy.body");
574 auto DoneBB = createBasicBlock("omp.arraycpy.done");
575 auto IsEmpty =
576 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
577 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000578
Alexey Bataev420d45b2015-04-14 05:11:24 +0000579 // Enter the loop body, making that address the current address.
580 auto EntryBB = Builder.GetInsertBlock();
581 EmitBlock(BodyBB);
John McCall7f416cc2015-09-08 08:05:57 +0000582
583 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
584
585 llvm::PHINode *SrcElementPHI =
586 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
587 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
588 Address SrcElementCurrent =
589 Address(SrcElementPHI,
590 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
591
592 llvm::PHINode *DestElementPHI =
593 Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
594 DestElementPHI->addIncoming(DestBegin, EntryBB);
595 Address DestElementCurrent =
596 Address(DestElementPHI,
597 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000598
Alexey Bataev420d45b2015-04-14 05:11:24 +0000599 // Emit copy.
600 CopyGen(DestElementCurrent, SrcElementCurrent);
601
602 // Shift the address forward by one element.
603 auto DestElementNext = Builder.CreateConstGEP1_32(
John McCall7f416cc2015-09-08 08:05:57 +0000604 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
Alexey Bataev420d45b2015-04-14 05:11:24 +0000605 auto SrcElementNext = Builder.CreateConstGEP1_32(
John McCall7f416cc2015-09-08 08:05:57 +0000606 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
Alexey Bataev420d45b2015-04-14 05:11:24 +0000607 // Check whether we've reached the end.
608 auto Done =
609 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
610 Builder.CreateCondBr(Done, DoneBB, BodyBB);
John McCall7f416cc2015-09-08 08:05:57 +0000611 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
612 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
Alexey Bataev420d45b2015-04-14 05:11:24 +0000613
614 // Done.
615 EmitBlock(DoneBB, /*IsFinished=*/true);
616}
617
John McCall7f416cc2015-09-08 08:05:57 +0000618void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
619 Address SrcAddr, const VarDecl *DestVD,
Alexey Bataev420d45b2015-04-14 05:11:24 +0000620 const VarDecl *SrcVD, const Expr *Copy) {
621 if (OriginalType->isArrayType()) {
622 auto *BO = dyn_cast<BinaryOperator>(Copy);
623 if (BO && BO->getOpcode() == BO_Assign) {
624 // Perform simple memcpy for simple copying.
John McCall7f416cc2015-09-08 08:05:57 +0000625 EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
Alexey Bataev420d45b2015-04-14 05:11:24 +0000626 } else {
627 // For arrays with complex element types perform element by element
628 // copying.
John McCall7f416cc2015-09-08 08:05:57 +0000629 EmitOMPAggregateAssign(
Alexey Bataev420d45b2015-04-14 05:11:24 +0000630 DestAddr, SrcAddr, OriginalType,
John McCall7f416cc2015-09-08 08:05:57 +0000631 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
Alexey Bataev420d45b2015-04-14 05:11:24 +0000632 // Working with the single array element, so have to remap
633 // destination and source variables to corresponding array
634 // elements.
John McCall7f416cc2015-09-08 08:05:57 +0000635 CodeGenFunction::OMPPrivateScope Remap(*this);
636 Remap.addPrivate(DestVD, [DestElement]() -> Address {
Alexey Bataev420d45b2015-04-14 05:11:24 +0000637 return DestElement;
638 });
639 Remap.addPrivate(
John McCall7f416cc2015-09-08 08:05:57 +0000640 SrcVD, [SrcElement]() -> Address { return SrcElement; });
Alexey Bataev420d45b2015-04-14 05:11:24 +0000641 (void)Remap.Privatize();
John McCall7f416cc2015-09-08 08:05:57 +0000642 EmitIgnoredExpr(Copy);
Alexey Bataev420d45b2015-04-14 05:11:24 +0000643 });
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000644 }
Alexey Bataev420d45b2015-04-14 05:11:24 +0000645 } else {
646 // Remap pseudo source variable to private copy.
John McCall7f416cc2015-09-08 08:05:57 +0000647 CodeGenFunction::OMPPrivateScope Remap(*this);
648 Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; });
649 Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; });
Alexey Bataev420d45b2015-04-14 05:11:24 +0000650 (void)Remap.Privatize();
651 // Emit copying of the whole variable.
John McCall7f416cc2015-09-08 08:05:57 +0000652 EmitIgnoredExpr(Copy);
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000653 }
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000654}
655
Alexey Bataev69c62a92015-04-15 04:52:20 +0000656bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
657 OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000658 if (!HaveInsertPoint())
659 return false;
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000660 bool FirstprivateIsLastprivate = false;
661 llvm::DenseSet<const VarDecl *> Lastprivates;
662 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
663 for (const auto *D : C->varlists())
664 Lastprivates.insert(
665 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
666 }
Alexey Bataev69c62a92015-04-15 04:52:20 +0000667 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
Alexey Bataev9afe5752016-05-24 07:40:12 +0000668 CGCapturedStmtInfo CapturesInfo(cast<CapturedStmt>(*D.getAssociatedStmt()));
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000669 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000670 auto IRef = C->varlist_begin();
671 auto InitsRef = C->inits().begin();
672 for (auto IInit : C->private_copies()) {
Alexey Bataev435ad7b2014-10-10 09:48:26 +0000673 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000674 bool ThisFirstprivateIsLastprivate =
675 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
Alexey Bataev9afe5752016-05-24 07:40:12 +0000676 auto *CapFD = CapturesInfo.lookup(OrigVD);
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000677 auto *FD = CapturedStmtInfo->lookup(OrigVD);
Alexey Bataev9afe5752016-05-24 07:40:12 +0000678 if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) &&
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000679 !FD->getType()->isReferenceType()) {
680 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
681 ++IRef;
682 ++InitsRef;
683 continue;
684 }
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000685 FirstprivateIsLastprivate =
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000686 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000687 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataev69c62a92015-04-15 04:52:20 +0000688 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
689 auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
690 bool IsRegistered;
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000691 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
692 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
693 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
John McCall7f416cc2015-09-08 08:05:57 +0000694 Address OriginalAddr = EmitLValue(&DRE).getAddress();
Alexey Bataevfeddd642016-04-22 09:05:03 +0000695 QualType Type = VD->getType();
Alexey Bataev1d9c15c2015-05-19 12:31:28 +0000696 if (Type->isArrayType()) {
Alexey Bataev69c62a92015-04-15 04:52:20 +0000697 // Emit VarDecl with copy init for arrays.
698 // Get the address of the original variable captured in current
699 // captured region.
John McCall7f416cc2015-09-08 08:05:57 +0000700 IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
Alexey Bataev69c62a92015-04-15 04:52:20 +0000701 auto Emission = EmitAutoVarAlloca(*VD);
702 auto *Init = VD->getInit();
703 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
704 // Perform simple memcpy.
705 EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
Alexey Bataev1d9c15c2015-05-19 12:31:28 +0000706 Type);
Alexey Bataev69c62a92015-04-15 04:52:20 +0000707 } else {
708 EmitOMPAggregateAssign(
Alexey Bataev1d9c15c2015-05-19 12:31:28 +0000709 Emission.getAllocatedAddress(), OriginalAddr, Type,
John McCall7f416cc2015-09-08 08:05:57 +0000710 [this, VDInit, Init](Address DestElement,
711 Address SrcElement) {
Alexey Bataev69c62a92015-04-15 04:52:20 +0000712 // Clean up any temporaries needed by the initialization.
713 RunCleanupsScope InitScope(*this);
714 // Emit initialization for single element.
John McCall7f416cc2015-09-08 08:05:57 +0000715 setAddrOfLocalVar(VDInit, SrcElement);
Alexey Bataev69c62a92015-04-15 04:52:20 +0000716 EmitAnyExprToMem(Init, DestElement,
717 Init->getType().getQualifiers(),
718 /*IsInitializer*/ false);
719 LocalDeclMap.erase(VDInit);
720 });
721 }
722 EmitAutoVarCleanups(Emission);
723 return Emission.getAllocatedAddress();
724 });
725 } else {
John McCall7f416cc2015-09-08 08:05:57 +0000726 IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
Alexey Bataev69c62a92015-04-15 04:52:20 +0000727 // Emit private VarDecl with copy init.
728 // Remap temp VDInit variable to the address of the original
729 // variable
730 // (for proper handling of captured global variables).
John McCall7f416cc2015-09-08 08:05:57 +0000731 setAddrOfLocalVar(VDInit, OriginalAddr);
Alexey Bataev69c62a92015-04-15 04:52:20 +0000732 EmitDecl(*VD);
733 LocalDeclMap.erase(VDInit);
734 return GetAddrOfLocalVar(VD);
735 });
736 }
737 assert(IsRegistered &&
738 "firstprivate var already registered as private");
739 // Silence the warning about unused variable.
740 (void)IsRegistered;
741 }
Richard Trieucc3949d2016-02-18 22:34:54 +0000742 ++IRef;
743 ++InitsRef;
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000744 }
745 }
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000746 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000747}
748
Alexey Bataev03b340a2014-10-21 03:16:40 +0000749void CodeGenFunction::EmitOMPPrivateClause(
750 const OMPExecutableDirective &D,
751 CodeGenFunction::OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000752 if (!HaveInsertPoint())
753 return;
Alexey Bataev50a64582015-04-22 12:24:45 +0000754 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000755 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
Alexey Bataev03b340a2014-10-21 03:16:40 +0000756 auto IRef = C->varlist_begin();
757 for (auto IInit : C->private_copies()) {
758 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev50a64582015-04-22 12:24:45 +0000759 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
760 auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
761 bool IsRegistered =
John McCall7f416cc2015-09-08 08:05:57 +0000762 PrivateScope.addPrivate(OrigVD, [&]() -> Address {
Alexey Bataev50a64582015-04-22 12:24:45 +0000763 // Emit private VarDecl with copy init.
764 EmitDecl(*VD);
765 return GetAddrOfLocalVar(VD);
766 });
767 assert(IsRegistered && "private var already registered as private");
768 // Silence the warning about unused variable.
769 (void)IsRegistered;
770 }
Alexey Bataev03b340a2014-10-21 03:16:40 +0000771 ++IRef;
772 }
773 }
774}
775
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000776bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000777 if (!HaveInsertPoint())
778 return false;
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000779 // threadprivate_var1 = master_threadprivate_var1;
780 // operator=(threadprivate_var2, master_threadprivate_var2);
781 // ...
782 // __kmpc_barrier(&loc, global_tid);
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000783 llvm::DenseSet<const VarDecl *> CopiedVars;
784 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000785 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000786 auto IRef = C->varlist_begin();
787 auto ISrcRef = C->source_exprs().begin();
788 auto IDestRef = C->destination_exprs().begin();
789 for (auto *AssignOp : C->assignment_ops()) {
790 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev1d9c15c2015-05-19 12:31:28 +0000791 QualType Type = VD->getType();
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000792 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000793 // Get the address of the master variable. If we are emitting code with
794 // TLS support, the address is passed from the master as field in the
795 // captured declaration.
John McCall7f416cc2015-09-08 08:05:57 +0000796 Address MasterAddr = Address::invalid();
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000797 if (getLangOpts().OpenMPUseTLS &&
798 getContext().getTargetInfo().isTLSSupported()) {
799 assert(CapturedStmtInfo->lookup(VD) &&
800 "Copyin threadprivates should have been captured!");
801 DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
802 VK_LValue, (*IRef)->getExprLoc());
803 MasterAddr = EmitLValue(&DRE).getAddress();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000804 LocalDeclMap.erase(VD);
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000805 } else {
John McCall7f416cc2015-09-08 08:05:57 +0000806 MasterAddr =
807 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
808 : CGM.GetAddrOfGlobal(VD),
809 getContext().getDeclAlign(VD));
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000810 }
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000811 // Get the address of the threadprivate variable.
John McCall7f416cc2015-09-08 08:05:57 +0000812 Address PrivateAddr = EmitLValue(*IRef).getAddress();
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000813 if (CopiedVars.size() == 1) {
814 // At first check if current thread is a master thread. If it is, no
815 // need to copy data.
816 CopyBegin = createBasicBlock("copyin.not.master");
817 CopyEnd = createBasicBlock("copyin.not.master.end");
818 Builder.CreateCondBr(
819 Builder.CreateICmpNE(
John McCall7f416cc2015-09-08 08:05:57 +0000820 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
821 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000822 CopyBegin, CopyEnd);
823 EmitBlock(CopyBegin);
824 }
825 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
826 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
John McCall7f416cc2015-09-08 08:05:57 +0000827 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000828 }
829 ++IRef;
830 ++ISrcRef;
831 ++IDestRef;
832 }
833 }
834 if (CopyEnd) {
835 // Exit out of copying procedure for non-master thread.
836 EmitBlock(CopyEnd, /*IsFinished=*/true);
837 return true;
838 }
839 return false;
840}
841
Alexey Bataev38e89532015-04-16 04:54:05 +0000842bool CodeGenFunction::EmitOMPLastprivateClauseInit(
843 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000844 if (!HaveInsertPoint())
845 return false;
Alexey Bataev38e89532015-04-16 04:54:05 +0000846 bool HasAtLeastOneLastprivate = false;
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000847 llvm::DenseSet<const VarDecl *> SIMDLCVs;
848 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
849 auto *LoopDirective = cast<OMPLoopDirective>(&D);
850 for (auto *C : LoopDirective->counters()) {
851 SIMDLCVs.insert(
852 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
853 }
854 }
Alexey Bataev38e89532015-04-16 04:54:05 +0000855 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000856 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
Alexey Bataevd130fd12015-05-13 10:23:02 +0000857 HasAtLeastOneLastprivate = true;
Alexey Bataevf93095a2016-05-05 08:46:22 +0000858 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()))
859 break;
Alexey Bataev38e89532015-04-16 04:54:05 +0000860 auto IRef = C->varlist_begin();
861 auto IDestRef = C->destination_exprs().begin();
862 for (auto *IInit : C->private_copies()) {
863 // Keep the address of the original variable for future update at the end
864 // of the loop.
865 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataevf93095a2016-05-05 08:46:22 +0000866 // Taskloops do not require additional initialization, it is done in
867 // runtime support library.
Alexey Bataev38e89532015-04-16 04:54:05 +0000868 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
869 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
John McCall7f416cc2015-09-08 08:05:57 +0000870 PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address {
Alexey Bataev38e89532015-04-16 04:54:05 +0000871 DeclRefExpr DRE(
872 const_cast<VarDecl *>(OrigVD),
873 /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
874 OrigVD) != nullptr,
875 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
876 return EmitLValue(&DRE).getAddress();
877 });
878 // Check if the variable is also a firstprivate: in this case IInit is
879 // not generated. Initialization of this variable will happen in codegen
880 // for 'firstprivate' clause.
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000881 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
Alexey Bataevd130fd12015-05-13 10:23:02 +0000882 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
Alexey Bataevf93095a2016-05-05 08:46:22 +0000883 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
884 // Emit private VarDecl with copy init.
885 EmitDecl(*VD);
886 return GetAddrOfLocalVar(VD);
887 });
Alexey Bataevd130fd12015-05-13 10:23:02 +0000888 assert(IsRegistered &&
889 "lastprivate var already registered as private");
890 (void)IsRegistered;
891 }
Alexey Bataev38e89532015-04-16 04:54:05 +0000892 }
Richard Trieucc3949d2016-02-18 22:34:54 +0000893 ++IRef;
894 ++IDestRef;
Alexey Bataev38e89532015-04-16 04:54:05 +0000895 }
896 }
897 return HasAtLeastOneLastprivate;
898}
899
900void CodeGenFunction::EmitOMPLastprivateClauseFinal(
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000901 const OMPExecutableDirective &D, bool NoFinals,
902 llvm::Value *IsLastIterCond) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000903 if (!HaveInsertPoint())
904 return;
Alexey Bataev38e89532015-04-16 04:54:05 +0000905 // Emit following code:
906 // if (<IsLastIterCond>) {
907 // orig_var1 = private_orig_var1;
908 // ...
909 // orig_varn = private_orig_varn;
910 // }
Alexey Bataevfc087ec2015-06-16 13:14:42 +0000911 llvm::BasicBlock *ThenBB = nullptr;
912 llvm::BasicBlock *DoneBB = nullptr;
913 if (IsLastIterCond) {
914 ThenBB = createBasicBlock(".omp.lastprivate.then");
915 DoneBB = createBasicBlock(".omp.lastprivate.done");
916 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
917 EmitBlock(ThenBB);
918 }
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000919 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
920 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
Alexey Bataev7a228ff2015-05-21 07:59:51 +0000921 if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
Alexey Bataev8ffcc942016-02-18 13:48:15 +0000922 auto IC = LoopDirective->counters().begin();
923 for (auto F : LoopDirective->finals()) {
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000924 auto *D =
925 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
926 if (NoFinals)
927 AlreadyEmittedVars.insert(D);
928 else
929 LoopCountersAndUpdates[D] = F;
Alexey Bataev8ffcc942016-02-18 13:48:15 +0000930 ++IC;
Alexey Bataev7a228ff2015-05-21 07:59:51 +0000931 }
932 }
Alexey Bataev8ffcc942016-02-18 13:48:15 +0000933 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
934 auto IRef = C->varlist_begin();
935 auto ISrcRef = C->source_exprs().begin();
936 auto IDestRef = C->destination_exprs().begin();
937 for (auto *AssignOp : C->assignment_ops()) {
938 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
939 QualType Type = PrivateVD->getType();
940 auto *CanonicalVD = PrivateVD->getCanonicalDecl();
941 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
942 // If lastprivate variable is a loop control variable for loop-based
943 // directive, update its value before copyin back to original
944 // variable.
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000945 if (auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
946 EmitIgnoredExpr(FinalExpr);
Alexey Bataev8ffcc942016-02-18 13:48:15 +0000947 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
948 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
949 // Get the address of the original variable.
950 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
951 // Get the address of the private variable.
952 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
953 if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>())
954 PrivateAddr =
John McCall7f416cc2015-09-08 08:05:57 +0000955 Address(Builder.CreateLoad(PrivateAddr),
956 getNaturalTypeAlignment(RefTy->getPointeeType()));
Alexey Bataev8ffcc942016-02-18 13:48:15 +0000957 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
Alexey Bataev38e89532015-04-16 04:54:05 +0000958 }
Alexey Bataev8ffcc942016-02-18 13:48:15 +0000959 ++IRef;
960 ++ISrcRef;
961 ++IDestRef;
Alexey Bataev38e89532015-04-16 04:54:05 +0000962 }
Alexey Bataev005248a2016-02-25 05:25:57 +0000963 if (auto *PostUpdate = C->getPostUpdateExpr())
964 EmitIgnoredExpr(PostUpdate);
Alexey Bataev38e89532015-04-16 04:54:05 +0000965 }
Alexey Bataev8ffcc942016-02-18 13:48:15 +0000966 if (IsLastIterCond)
Alexey Bataevfc087ec2015-06-16 13:14:42 +0000967 EmitBlock(DoneBB, /*IsFinished=*/true);
Alexey Bataev38e89532015-04-16 04:54:05 +0000968}
969
Alexey Bataev794ba0d2015-04-10 10:43:45 +0000970void CodeGenFunction::EmitOMPReductionClauseInit(
971 const OMPExecutableDirective &D,
972 CodeGenFunction::OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000973 if (!HaveInsertPoint())
974 return;
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000975 SmallVector<const Expr *, 4> Shareds;
976 SmallVector<const Expr *, 4> Privates;
977 SmallVector<const Expr *, 4> ReductionOps;
978 SmallVector<const Expr *, 4> LHSs;
979 SmallVector<const Expr *, 4> RHSs;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000980 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
Alexey Bataevf24e7b12015-10-08 09:10:53 +0000981 auto IPriv = C->privates().begin();
Alexey Bataeva839ddd2016-03-17 10:19:46 +0000982 auto IRed = C->reduction_ops().begin();
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000983 auto ILHS = C->lhs_exprs().begin();
984 auto IRHS = C->rhs_exprs().begin();
985 for (const auto *Ref : C->varlists()) {
986 Shareds.emplace_back(Ref);
987 Privates.emplace_back(*IPriv);
988 ReductionOps.emplace_back(*IRed);
989 LHSs.emplace_back(*ILHS);
990 RHSs.emplace_back(*IRHS);
991 std::advance(IPriv, 1);
992 std::advance(IRed, 1);
993 std::advance(ILHS, 1);
994 std::advance(IRHS, 1);
Alexey Bataev794ba0d2015-04-10 10:43:45 +0000995 }
996 }
Alexey Bataev5c40bec2017-07-13 13:36:14 +0000997 ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
998 unsigned Count = 0;
999 auto ILHS = LHSs.begin();
1000 auto IRHS = RHSs.begin();
1001 auto IPriv = Privates.begin();
1002 for (const auto *IRef : Shareds) {
1003 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1004 // Emit private VarDecl with reduction init.
1005 RedCG.emitSharedLValue(*this, Count);
1006 RedCG.emitAggregateType(*this, Count);
1007 auto Emission = EmitAutoVarAlloca(*PrivateVD);
1008 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1009 RedCG.getSharedLValue(Count),
1010 [&Emission](CodeGenFunction &CGF) {
1011 CGF.EmitAutoVarInit(Emission);
1012 return true;
1013 });
1014 EmitAutoVarCleanups(Emission);
1015 Address BaseAddr = RedCG.adjustPrivateAddress(
1016 *this, Count, Emission.getAllocatedAddress());
1017 bool IsRegistered = PrivateScope.addPrivate(
1018 RedCG.getBaseDecl(Count), [BaseAddr]() -> Address { return BaseAddr; });
1019 assert(IsRegistered && "private var already registered as private");
1020 // Silence the warning about unused variable.
1021 (void)IsRegistered;
1022
1023 auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1024 auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
Jonas Hahnfeld4525c822017-10-23 19:01:35 +00001025 QualType Type = PrivateVD->getType();
1026 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1027 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001028 // Store the address of the original variable associated with the LHS
1029 // implicit variable.
1030 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1031 return RedCG.getSharedLValue(Count).getAddress();
1032 });
1033 PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
1034 return GetAddrOfLocalVar(PrivateVD);
1035 });
Jonas Hahnfeld4525c822017-10-23 19:01:35 +00001036 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1037 isa<ArraySubscriptExpr>(IRef)) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001038 // Store the address of the original variable associated with the LHS
1039 // implicit variable.
1040 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1041 return RedCG.getSharedLValue(Count).getAddress();
1042 });
1043 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address {
1044 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1045 ConvertTypeForMem(RHSVD->getType()),
1046 "rhs.begin");
1047 });
1048 } else {
1049 QualType Type = PrivateVD->getType();
1050 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1051 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1052 // Store the address of the original variable associated with the LHS
1053 // implicit variable.
1054 if (IsArray) {
1055 OriginalAddr = Builder.CreateElementBitCast(
1056 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1057 }
1058 PrivateScope.addPrivate(
1059 LHSVD, [OriginalAddr]() -> Address { return OriginalAddr; });
1060 PrivateScope.addPrivate(
1061 RHSVD, [this, PrivateVD, RHSVD, IsArray]() -> Address {
1062 return IsArray
1063 ? Builder.CreateElementBitCast(
1064 GetAddrOfLocalVar(PrivateVD),
1065 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1066 : GetAddrOfLocalVar(PrivateVD);
1067 });
1068 }
1069 ++ILHS;
1070 ++IRHS;
1071 ++IPriv;
1072 ++Count;
1073 }
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001074}
1075
1076void CodeGenFunction::EmitOMPReductionClauseFinal(
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001077 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001078 if (!HaveInsertPoint())
1079 return;
Alexey Bataevf24e7b12015-10-08 09:10:53 +00001080 llvm::SmallVector<const Expr *, 8> Privates;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001081 llvm::SmallVector<const Expr *, 8> LHSExprs;
1082 llvm::SmallVector<const Expr *, 8> RHSExprs;
1083 llvm::SmallVector<const Expr *, 8> ReductionOps;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001084 bool HasAtLeastOneReduction = false;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001085 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001086 HasAtLeastOneReduction = true;
Alexey Bataevf24e7b12015-10-08 09:10:53 +00001087 Privates.append(C->privates().begin(), C->privates().end());
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001088 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1089 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1090 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1091 }
1092 if (HasAtLeastOneReduction) {
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001093 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1094 isOpenMPParallelDirective(D.getDirectiveKind()) ||
1095 D.getDirectiveKind() == OMPD_simd;
Alexey Bataev617db5f2017-12-04 15:38:33 +00001096 bool SimpleReduction = D.getDirectiveKind() == OMPD_simd ||
1097 D.getDirectiveKind() == OMPD_distribute_simd;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001098 // Emit nowait reduction if nowait clause is present or directive is a
1099 // parallel directive (it always has implicit barrier).
1100 CGM.getOpenMPRuntime().emitReduction(
Alexey Bataevf24e7b12015-10-08 09:10:53 +00001101 *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001102 {WithNowait, SimpleReduction, ReductionKind});
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001103 }
1104}
1105
Alexey Bataev61205072016-03-02 04:57:40 +00001106static void emitPostUpdateForReductionClause(
1107 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1108 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1109 if (!CGF.HaveInsertPoint())
1110 return;
1111 llvm::BasicBlock *DoneBB = nullptr;
1112 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1113 if (auto *PostUpdate = C->getPostUpdateExpr()) {
1114 if (!DoneBB) {
1115 if (auto *Cond = CondGen(CGF)) {
1116 // If the first post-update expression is found, emit conditional
1117 // block if it was requested.
1118 auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1119 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1120 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1121 CGF.EmitBlock(ThenBB);
1122 }
1123 }
1124 CGF.EmitIgnoredExpr(PostUpdate);
1125 }
1126 }
1127 if (DoneBB)
1128 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1129}
1130
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001131namespace {
1132/// Codegen lambda for appending distribute lower and upper bounds to outlined
1133/// parallel function. This is necessary for combined constructs such as
1134/// 'distribute parallel for'
1135typedef llvm::function_ref<void(CodeGenFunction &,
1136 const OMPExecutableDirective &,
1137 llvm::SmallVectorImpl<llvm::Value *> &)>
1138 CodeGenBoundParametersTy;
1139} // anonymous namespace
1140
1141static void emitCommonOMPParallelDirective(
1142 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1143 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1144 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00001145 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1146 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1147 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001148 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
Alexey Bataev1d677132015-04-22 13:57:31 +00001149 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
Alexey Bataev1d677132015-04-22 13:57:31 +00001150 auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1151 /*IgnoreResultAssign*/ true);
1152 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1153 CGF, NumThreads, NumThreadsClause->getLocStart());
1154 }
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001155 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001156 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
Alexey Bataev7f210c62015-06-18 13:40:03 +00001157 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1158 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1159 }
Alexey Bataev1d677132015-04-22 13:57:31 +00001160 const Expr *IfCond = nullptr;
Alexey Bataev7371aa32015-09-03 08:45:56 +00001161 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1162 if (C->getNameModifier() == OMPD_unknown ||
1163 C->getNameModifier() == OMPD_parallel) {
1164 IfCond = C->getCondition();
1165 break;
1166 }
Alexey Bataev1d677132015-04-22 13:57:31 +00001167 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001168
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +00001169 OMPParallelScope Scope(CGF, S);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001170 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001171 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1172 // lower and upper bounds with the pragma 'for' chunking mechanism.
1173 // The following lambda takes care of appending the lower and upper bound
1174 // parameters when necessary
1175 CodeGenBoundParameters(CGF, S, CapturedVars);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001176 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
Alexey Bataev1d677132015-04-22 13:57:31 +00001177 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
Alexey Bataev2377fe92015-09-10 08:12:02 +00001178 CapturedVars, IfCond);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001179}
1180
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001181static void emitEmptyBoundParameters(CodeGenFunction &,
1182 const OMPExecutableDirective &,
1183 llvm::SmallVectorImpl<llvm::Value *> &) {}
1184
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001185void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001186 // Emit parallel region as a standalone region.
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001187 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001188 OMPPrivateScope PrivateScope(CGF);
Alexey Bataevf56f98c2015-04-16 05:39:01 +00001189 bool Copyins = CGF.EmitOMPCopyinClause(S);
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00001190 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1191 if (Copyins) {
Alexey Bataev69c62a92015-04-15 04:52:20 +00001192 // Emit implicit barrier to synchronize threads and avoid data races on
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00001193 // propagation master's thread values of threadprivate variables to local
1194 // instances of that variables of all other implicit threads.
Alexey Bataev25e5b442015-09-15 12:52:43 +00001195 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1196 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1197 /*ForceSimpleCall=*/true);
Alexey Bataev69c62a92015-04-15 04:52:20 +00001198 }
1199 CGF.EmitOMPPrivateClause(S, PrivateScope);
1200 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1201 (void)PrivateScope.Privatize();
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001202 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001203 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001204 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001205 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1206 emitEmptyBoundParameters);
Alexey Bataev61205072016-03-02 04:57:40 +00001207 emitPostUpdateForReductionClause(
1208 *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
Alexey Bataev9959db52014-05-06 10:08:46 +00001209}
Alexander Musman515ad8c2014-05-22 08:54:05 +00001210
Alexey Bataev0f34da12015-07-02 04:17:07 +00001211void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1212 JumpDest LoopExit) {
Alexander Musmana5f070a2014-10-01 06:03:56 +00001213 RunCleanupsScope BodyScope(*this);
1214 // Update counters values on current iteration.
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001215 for (auto I : D.updates()) {
Alexander Musmana5f070a2014-10-01 06:03:56 +00001216 EmitIgnoredExpr(I);
1217 }
Alexander Musman3276a272015-03-21 10:12:56 +00001218 // Update the linear variables.
Alexey Bataev617db5f2017-12-04 15:38:33 +00001219 // In distribute directives only loop counters may be marked as linear, no
1220 // need to generate the code for them.
1221 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1222 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1223 for (auto *U : C->updates())
1224 EmitIgnoredExpr(U);
1225 }
Alexander Musman3276a272015-03-21 10:12:56 +00001226 }
1227
Alexander Musmana5f070a2014-10-01 06:03:56 +00001228 // On a continue in the body, jump to the end.
Alexander Musmand196ef22014-10-07 08:57:09 +00001229 auto Continue = getJumpDestInCurrentScope("omp.body.continue");
Alexey Bataev0f34da12015-07-02 04:17:07 +00001230 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
Alexander Musmana5f070a2014-10-01 06:03:56 +00001231 // Emit loop body.
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001232 EmitStmt(D.getBody());
Alexander Musmana5f070a2014-10-01 06:03:56 +00001233 // The end (updates/cleanups).
1234 EmitBlock(Continue.getBlock());
1235 BreakContinueStack.pop_back();
Alexander Musmana5f070a2014-10-01 06:03:56 +00001236}
1237
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001238void CodeGenFunction::EmitOMPInnerLoop(
1239 const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1240 const Expr *IncExpr,
Alexey Bataev98eb6e32015-04-22 11:15:40 +00001241 const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
1242 const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
Alexander Musmand196ef22014-10-07 08:57:09 +00001243 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001244
1245 // Start the loop with a block that tests the condition.
Alexander Musmand196ef22014-10-07 08:57:09 +00001246 auto CondBlock = createBasicBlock("omp.inner.for.cond");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001247 EmitBlock(CondBlock);
Amara Emerson652795d2016-11-10 14:44:30 +00001248 const SourceRange &R = S.getSourceRange();
1249 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1250 SourceLocToDebugLoc(R.getEnd()));
Alexander Musmana5f070a2014-10-01 06:03:56 +00001251
1252 // If there are any cleanups between here and the loop-exit scope,
1253 // create a block to stage a loop exit along.
1254 auto ExitBlock = LoopExit.getBlock();
Alexey Bataev2df54a02015-03-12 08:53:29 +00001255 if (RequiresCleanup)
Alexander Musmand196ef22014-10-07 08:57:09 +00001256 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001257
Alexander Musmand196ef22014-10-07 08:57:09 +00001258 auto LoopBody = createBasicBlock("omp.inner.for.body");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001259
Alexey Bataev2df54a02015-03-12 08:53:29 +00001260 // Emit condition.
Justin Bogner66242d62015-04-23 23:06:47 +00001261 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
Alexander Musmana5f070a2014-10-01 06:03:56 +00001262 if (ExitBlock != LoopExit.getBlock()) {
1263 EmitBlock(ExitBlock);
1264 EmitBranchThroughCleanup(LoopExit);
1265 }
1266
1267 EmitBlock(LoopBody);
Justin Bogner66242d62015-04-23 23:06:47 +00001268 incrementProfileCounter(&S);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001269
1270 // Create a block for the increment.
Alexander Musmand196ef22014-10-07 08:57:09 +00001271 auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001272 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1273
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001274 BodyGen(*this);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001275
1276 // Emit "IV = IV + 1" and a back-edge to the condition block.
1277 EmitBlock(Continue.getBlock());
Alexey Bataev2df54a02015-03-12 08:53:29 +00001278 EmitIgnoredExpr(IncExpr);
Alexey Bataev98eb6e32015-04-22 11:15:40 +00001279 PostIncGen(*this);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001280 BreakContinueStack.pop_back();
1281 EmitBranch(CondBlock);
1282 LoopStack.pop();
1283 // Emit the fall-through block.
1284 EmitBlock(LoopExit.getBlock());
1285}
1286
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001287bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001288 if (!HaveInsertPoint())
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001289 return false;
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001290 // Emit inits for the linear variables.
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001291 bool HasLinears = false;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001292 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001293 for (auto *Init : C->inits()) {
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001294 HasLinears = true;
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001295 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
Alexey Bataevef549a82016-03-09 09:49:09 +00001296 if (auto *Ref = dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1297 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1298 auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1299 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1300 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1301 VD->getInit()->getType(), VK_LValue,
1302 VD->getInit()->getExprLoc());
1303 EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1304 VD->getType()),
1305 /*capturedByInit=*/false);
1306 EmitAutoVarCleanups(Emission);
1307 } else
1308 EmitVarDecl(*VD);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001309 }
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001310 // Emit the linear steps for the linear clauses.
1311 // If a step is not constant, it is pre-calculated before the loop.
1312 if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1313 if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00001314 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001315 // Emit calculation of the linear step.
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00001316 EmitIgnoredExpr(CS);
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001317 }
Alexander Musmana5f070a2014-10-01 06:03:56 +00001318 }
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001319 return HasLinears;
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001320}
1321
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001322void CodeGenFunction::EmitOMPLinearClauseFinal(
1323 const OMPLoopDirective &D,
Alexey Bataevef549a82016-03-09 09:49:09 +00001324 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001325 if (!HaveInsertPoint())
Alexey Bataev8ef31412015-12-18 07:58:25 +00001326 return;
Alexey Bataevef549a82016-03-09 09:49:09 +00001327 llvm::BasicBlock *DoneBB = nullptr;
Alexander Musman3276a272015-03-21 10:12:56 +00001328 // Emit the final values of the linear variables.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001329 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
Alexey Bataev39f915b82015-05-08 10:41:21 +00001330 auto IC = C->varlist_begin();
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001331 for (auto *F : C->finals()) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001332 if (!DoneBB) {
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001333 if (auto *Cond = CondGen(*this)) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001334 // If the first post-update expression is found, emit conditional
1335 // block if it was requested.
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001336 auto *ThenBB = createBasicBlock(".omp.linear.pu");
1337 DoneBB = createBasicBlock(".omp.linear.pu.done");
1338 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1339 EmitBlock(ThenBB);
Alexey Bataevef549a82016-03-09 09:49:09 +00001340 }
1341 }
Alexey Bataev39f915b82015-05-08 10:41:21 +00001342 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1343 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001344 CapturedStmtInfo->lookup(OrigVD) != nullptr,
Alexey Bataev39f915b82015-05-08 10:41:21 +00001345 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001346 Address OrigAddr = EmitLValue(&DRE).getAddress();
1347 CodeGenFunction::OMPPrivateScope VarScope(*this);
1348 VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; });
Alexey Bataev39f915b82015-05-08 10:41:21 +00001349 (void)VarScope.Privatize();
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001350 EmitIgnoredExpr(F);
Alexey Bataev39f915b82015-05-08 10:41:21 +00001351 ++IC;
Alexander Musman3276a272015-03-21 10:12:56 +00001352 }
Alexey Bataev78849fb2016-03-09 09:49:00 +00001353 if (auto *PostUpdate = C->getPostUpdateExpr())
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001354 EmitIgnoredExpr(PostUpdate);
Alexander Musman3276a272015-03-21 10:12:56 +00001355 }
Alexey Bataevef549a82016-03-09 09:49:09 +00001356 if (DoneBB)
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001357 EmitBlock(DoneBB, /*IsFinished=*/true);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001358}
1359
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001360static void emitAlignedClause(CodeGenFunction &CGF,
1361 const OMPExecutableDirective &D) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001362 if (!CGF.HaveInsertPoint())
1363 return;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001364 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001365 unsigned ClauseAlignment = 0;
1366 if (auto AlignmentExpr = Clause->getAlignment()) {
1367 auto AlignmentCI =
1368 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1369 ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
Alexander Musman09184fe2014-09-30 05:29:28 +00001370 }
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001371 for (auto E : Clause->varlists()) {
1372 unsigned Alignment = ClauseAlignment;
1373 if (Alignment == 0) {
1374 // OpenMP [2.8.1, Description]
1375 // If no optional parameter is specified, implementation-defined default
1376 // alignments for SIMD instructions on the target platforms are assumed.
1377 Alignment =
Alexey Bataev00396512015-07-02 03:40:19 +00001378 CGF.getContext()
1379 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1380 E->getType()->getPointeeType()))
1381 .getQuantity();
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001382 }
1383 assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1384 "alignment is not power of 2");
1385 if (Alignment != 0) {
1386 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1387 CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1388 }
Alexander Musman09184fe2014-09-30 05:29:28 +00001389 }
1390 }
1391}
1392
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001393void CodeGenFunction::EmitOMPPrivateLoopCounters(
1394 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1395 if (!HaveInsertPoint())
Alexey Bataev8ef31412015-12-18 07:58:25 +00001396 return;
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001397 auto I = S.private_counters().begin();
1398 for (auto *E : S.counters()) {
Alexey Bataeva8899172015-08-06 12:30:57 +00001399 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1400 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001401 (void)LoopScope.addPrivate(VD, [&]() -> Address {
Alexey Bataev435ad7b2014-10-10 09:48:26 +00001402 // Emit var without initialization.
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001403 if (!LocalDeclMap.count(PrivateVD)) {
1404 auto VarEmission = EmitAutoVarAlloca(*PrivateVD);
1405 EmitAutoVarCleanups(VarEmission);
1406 }
1407 DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1408 /*RefersToEnclosingVariableOrCapture=*/false,
1409 (*I)->getType(), VK_LValue, (*I)->getExprLoc());
1410 return EmitLValue(&DRE).getAddress();
Alexey Bataev435ad7b2014-10-10 09:48:26 +00001411 });
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001412 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1413 VD->hasGlobalStorage()) {
1414 (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address {
1415 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1416 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1417 E->getType(), VK_LValue, E->getExprLoc());
1418 return EmitLValue(&DRE).getAddress();
1419 });
1420 }
Alexey Bataeva8899172015-08-06 12:30:57 +00001421 ++I;
Alexey Bataev435ad7b2014-10-10 09:48:26 +00001422 }
Alexey Bataev435ad7b2014-10-10 09:48:26 +00001423}
1424
Alexey Bataev62dbb972015-04-22 11:59:37 +00001425static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1426 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1427 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001428 if (!CGF.HaveInsertPoint())
1429 return;
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001430 {
1431 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001432 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001433 (void)PreCondScope.Privatize();
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001434 // Get initial values of real counters.
Alexey Bataevb08f89f2015-08-14 12:25:37 +00001435 for (auto I : S.inits()) {
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001436 CGF.EmitIgnoredExpr(I);
1437 }
Alexey Bataev62dbb972015-04-22 11:59:37 +00001438 }
1439 // Check that loop is executed at least one time.
1440 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1441}
1442
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001443void CodeGenFunction::EmitOMPLinearClause(
1444 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1445 if (!HaveInsertPoint())
Alexey Bataev8ef31412015-12-18 07:58:25 +00001446 return;
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001447 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1448 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1449 auto *LoopDirective = cast<OMPLoopDirective>(&D);
1450 for (auto *C : LoopDirective->counters()) {
1451 SIMDLCVs.insert(
1452 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1453 }
1454 }
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001455 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
Alexey Bataevbd9fec12015-08-18 06:47:21 +00001456 auto CurPrivate = C->privates().begin();
Alexey Bataevc925aa32015-04-27 08:00:32 +00001457 for (auto *E : C->varlists()) {
Alexey Bataevbd9fec12015-08-18 06:47:21 +00001458 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1459 auto *PrivateVD =
1460 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001461 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1462 bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address {
1463 // Emit private VarDecl with copy init.
1464 EmitVarDecl(*PrivateVD);
1465 return GetAddrOfLocalVar(PrivateVD);
1466 });
1467 assert(IsRegistered && "linear var already registered as private");
1468 // Silence the warning about unused variable.
1469 (void)IsRegistered;
1470 } else
1471 EmitVarDecl(*PrivateVD);
Alexey Bataevbd9fec12015-08-18 06:47:21 +00001472 ++CurPrivate;
Alexander Musman3276a272015-03-21 10:12:56 +00001473 }
1474 }
1475}
1476
Alexey Bataev45bfad52015-08-21 12:19:04 +00001477static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001478 const OMPExecutableDirective &D,
1479 bool IsMonotonic) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001480 if (!CGF.HaveInsertPoint())
1481 return;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001482 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
Alexey Bataev45bfad52015-08-21 12:19:04 +00001483 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1484 /*ignoreResult=*/true);
1485 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1486 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1487 // In presence of finite 'safelen', it may be unsafe to mark all
1488 // the memory instructions parallel, because loop-carried
1489 // dependences of 'safelen' iterations are possible.
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001490 if (!IsMonotonic)
1491 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001492 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001493 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1494 /*ignoreResult=*/true);
1495 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
Tyler Nowickida46d0e2015-07-14 23:03:09 +00001496 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001497 // In presence of finite 'safelen', it may be unsafe to mark all
1498 // the memory instructions parallel, because loop-carried
1499 // dependences of 'safelen' iterations are possible.
1500 CGF.LoopStack.setParallel(false);
1501 }
1502}
1503
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001504void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1505 bool IsMonotonic) {
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001506 // Walk clauses and process safelen/lastprivate.
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001507 LoopStack.setParallel(!IsMonotonic);
Tyler Nowickida46d0e2015-07-14 23:03:09 +00001508 LoopStack.setVectorizeEnable(true);
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001509 emitSimdlenSafelenClause(*this, D, IsMonotonic);
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001510}
1511
Alexey Bataevef549a82016-03-09 09:49:09 +00001512void CodeGenFunction::EmitOMPSimdFinal(
1513 const OMPLoopDirective &D,
1514 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001515 if (!HaveInsertPoint())
1516 return;
Alexey Bataevef549a82016-03-09 09:49:09 +00001517 llvm::BasicBlock *DoneBB = nullptr;
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001518 auto IC = D.counters().begin();
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001519 auto IPC = D.private_counters().begin();
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001520 for (auto F : D.finals()) {
1521 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001522 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1523 auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1524 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1525 OrigVD->hasGlobalStorage() || CED) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001526 if (!DoneBB) {
1527 if (auto *Cond = CondGen(*this)) {
1528 // If the first post-update expression is found, emit conditional
1529 // block if it was requested.
1530 auto *ThenBB = createBasicBlock(".omp.final.then");
1531 DoneBB = createBasicBlock(".omp.final.done");
1532 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1533 EmitBlock(ThenBB);
1534 }
1535 }
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001536 Address OrigAddr = Address::invalid();
1537 if (CED)
1538 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1539 else {
1540 DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1541 /*RefersToEnclosingVariableOrCapture=*/false,
1542 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1543 OrigAddr = EmitLValue(&DRE).getAddress();
1544 }
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001545 OMPPrivateScope VarScope(*this);
1546 VarScope.addPrivate(OrigVD,
John McCall7f416cc2015-09-08 08:05:57 +00001547 [OrigAddr]() -> Address { return OrigAddr; });
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001548 (void)VarScope.Privatize();
1549 EmitIgnoredExpr(F);
1550 }
1551 ++IC;
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001552 ++IPC;
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001553 }
Alexey Bataevef549a82016-03-09 09:49:09 +00001554 if (DoneBB)
1555 EmitBlock(DoneBB, /*IsFinished=*/true);
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001556}
1557
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001558static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1559 const OMPLoopDirective &S,
1560 CodeGenFunction::JumpDest LoopExit) {
1561 CGF.EmitOMPLoopBody(S, LoopExit);
1562 CGF.EmitStopPoint(&S);
Hans Wennborged129ae2017-04-27 17:02:25 +00001563}
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001564
Alexey Bataevf8365372017-11-17 17:57:25 +00001565static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
1566 PrePostActionTy &Action) {
1567 Action.Enter(CGF);
1568 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
1569 "Expected simd directive");
1570 OMPLoopScope PreInitScope(CGF, S);
1571 // if (PreCond) {
1572 // for (IV in 0..LastIteration) BODY;
1573 // <Final counter/linear vars updates>;
1574 // }
1575 //
Alexander Musmana5f070a2014-10-01 06:03:56 +00001576
Alexey Bataevf8365372017-11-17 17:57:25 +00001577 // Emit: if (PreCond) - begin.
1578 // If the condition constant folds and can be elided, avoid emitting the
1579 // whole loop.
1580 bool CondConstant;
1581 llvm::BasicBlock *ContBlock = nullptr;
1582 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1583 if (!CondConstant)
1584 return;
1585 } else {
1586 auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
1587 ContBlock = CGF.createBasicBlock("simd.if.end");
1588 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1589 CGF.getProfileCount(&S));
1590 CGF.EmitBlock(ThenBlock);
1591 CGF.incrementProfileCounter(&S);
1592 }
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001593
Alexey Bataevf8365372017-11-17 17:57:25 +00001594 // Emit the loop iteration variable.
1595 const Expr *IVExpr = S.getIterationVariable();
1596 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1597 CGF.EmitVarDecl(*IVDecl);
1598 CGF.EmitIgnoredExpr(S.getInit());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001599
Alexey Bataevf8365372017-11-17 17:57:25 +00001600 // Emit the iterations count variable.
1601 // If it is not a variable, Sema decided to calculate iterations count on
1602 // each iteration (e.g., it is foldable into a constant).
1603 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1604 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1605 // Emit calculation of the iterations count.
1606 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1607 }
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001608
Alexey Bataevf8365372017-11-17 17:57:25 +00001609 CGF.EmitOMPSimdInit(S);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001610
Alexey Bataevf8365372017-11-17 17:57:25 +00001611 emitAlignedClause(CGF, S);
1612 (void)CGF.EmitOMPLinearClauseInit(S);
1613 {
1614 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1615 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1616 CGF.EmitOMPLinearClause(S, LoopScope);
1617 CGF.EmitOMPPrivateClause(S, LoopScope);
1618 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1619 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1620 (void)LoopScope.Privatize();
1621 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1622 S.getInc(),
1623 [&S](CodeGenFunction &CGF) {
1624 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
1625 CGF.EmitStopPoint(&S);
1626 },
1627 [](CodeGenFunction &) {});
1628 CGF.EmitOMPSimdFinal(
Alexey Bataevef549a82016-03-09 09:49:09 +00001629 S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
Alexey Bataevf8365372017-11-17 17:57:25 +00001630 // Emit final copy of the lastprivate variables at the end of loops.
1631 if (HasLastprivateClause)
1632 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1633 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1634 emitPostUpdateForReductionClause(
1635 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1636 }
1637 CGF.EmitOMPLinearClauseFinal(
1638 S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1639 // Emit: if (PreCond) - end.
1640 if (ContBlock) {
1641 CGF.EmitBranch(ContBlock);
1642 CGF.EmitBlock(ContBlock, true);
1643 }
1644}
1645
1646void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
1647 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1648 emitOMPSimdRegion(CGF, S, Action);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001649 };
Alexey Bataev4ba78a42016-04-27 07:56:03 +00001650 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00001651 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
Alexander Musman515ad8c2014-05-22 08:54:05 +00001652}
1653
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001654void CodeGenFunction::EmitOMPOuterLoop(
1655 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1656 CodeGenFunction::OMPPrivateScope &LoopScope,
1657 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1658 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1659 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001660 auto &RT = CGM.getOpenMPRuntime();
Alexander Musman92bdaab2015-03-12 13:37:50 +00001661
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001662 const Expr *IVExpr = S.getIterationVariable();
1663 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1664 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1665
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001666 auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1667
1668 // Start the loop with a block that tests the condition.
1669 auto CondBlock = createBasicBlock("omp.dispatch.cond");
1670 EmitBlock(CondBlock);
Amara Emerson652795d2016-11-10 14:44:30 +00001671 const SourceRange &R = S.getSourceRange();
1672 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1673 SourceLocToDebugLoc(R.getEnd()));
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001674
1675 llvm::Value *BoolCondVal = nullptr;
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00001676 if (!DynamicOrOrdered) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001677 // UB = min(UB, GlobalUB) or
1678 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1679 // 'distribute parallel for')
1680 EmitIgnoredExpr(LoopArgs.EUB);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001681 // IV = LB
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001682 EmitIgnoredExpr(LoopArgs.Init);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001683 // IV < UB
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001684 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001685 } else {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001686 BoolCondVal =
1687 RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1688 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001689 }
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001690
1691 // If there are any cleanups between here and the loop-exit scope,
1692 // create a block to stage a loop exit along.
1693 auto ExitBlock = LoopExit.getBlock();
1694 if (LoopScope.requiresCleanups())
1695 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1696
1697 auto LoopBody = createBasicBlock("omp.dispatch.body");
1698 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1699 if (ExitBlock != LoopExit.getBlock()) {
1700 EmitBlock(ExitBlock);
1701 EmitBranchThroughCleanup(LoopExit);
1702 }
1703 EmitBlock(LoopBody);
1704
Alexander Musman92bdaab2015-03-12 13:37:50 +00001705 // Emit "IV = LB" (in case of static schedule, we have already calculated new
1706 // LB for loop condition and emitted it above).
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00001707 if (DynamicOrOrdered)
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001708 EmitIgnoredExpr(LoopArgs.Init);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001709
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001710 // Create a block for the increment.
1711 auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1712 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1713
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001714 // Generate !llvm.loop.parallel metadata for loads and stores for loops
1715 // with dynamic/guided scheduling and without ordered clause.
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001716 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
1717 LoopStack.setParallel(!IsMonotonic);
1718 else
1719 EmitOMPSimdInit(S, IsMonotonic);
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001720
Alexey Bataev98eb6e32015-04-22 11:15:40 +00001721 SourceLocation Loc = S.getLocStart();
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001722
1723 // when 'distribute' is not combined with a 'for':
1724 // while (idx <= UB) { BODY; ++idx; }
1725 // when 'distribute' is combined with a 'for'
1726 // (e.g. 'distribute parallel for')
1727 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1728 EmitOMPInnerLoop(
1729 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1730 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1731 CodeGenLoop(CGF, S, LoopExit);
1732 },
1733 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1734 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1735 });
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001736
1737 EmitBlock(Continue.getBlock());
1738 BreakContinueStack.pop_back();
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00001739 if (!DynamicOrOrdered) {
Alexander Musman92bdaab2015-03-12 13:37:50 +00001740 // Emit "LB = LB + Stride", "UB = UB + Stride".
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001741 EmitIgnoredExpr(LoopArgs.NextLB);
1742 EmitIgnoredExpr(LoopArgs.NextUB);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001743 }
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001744
1745 EmitBranch(CondBlock);
1746 LoopStack.pop();
1747 // Emit the fall-through block.
1748 EmitBlock(LoopExit.getBlock());
1749
1750 // Tell the runtime we are done.
Alexey Bataev957d8562016-11-17 15:12:05 +00001751 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1752 if (!DynamicOrOrdered)
Alexey Bataevf43f7142017-09-06 16:17:35 +00001753 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
1754 S.getDirectiveKind());
Alexey Bataev957d8562016-11-17 15:12:05 +00001755 };
1756 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001757}
1758
1759void CodeGenFunction::EmitOMPForOuterLoop(
Alexey Bataev9ebd7422016-05-10 09:57:36 +00001760 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001761 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001762 const OMPLoopArguments &LoopArgs,
1763 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001764 auto &RT = CGM.getOpenMPRuntime();
1765
1766 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
Alexey Bataev9ebd7422016-05-10 09:57:36 +00001767 const bool DynamicOrOrdered =
1768 Ordered || RT.isDynamic(ScheduleKind.Schedule);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001769
1770 assert((Ordered ||
Alexey Bataev9ebd7422016-05-10 09:57:36 +00001771 !RT.isStaticNonchunked(ScheduleKind.Schedule,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001772 LoopArgs.Chunk != nullptr)) &&
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001773 "static non-chunked schedule does not need outer loop");
1774
1775 // Emit outer loop.
1776 //
1777 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1778 // When schedule(dynamic,chunk_size) is specified, the iterations are
1779 // distributed to threads in the team in chunks as the threads request them.
1780 // Each thread executes a chunk of iterations, then requests another chunk,
1781 // until no chunks remain to be distributed. Each chunk contains chunk_size
1782 // iterations, except for the last chunk to be distributed, which may have
1783 // fewer iterations. When no chunk_size is specified, it defaults to 1.
1784 //
1785 // When schedule(guided,chunk_size) is specified, the iterations are assigned
1786 // to threads in the team in chunks as the executing threads request them.
1787 // Each thread executes a chunk of iterations, then requests another chunk,
1788 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1789 // each chunk is proportional to the number of unassigned iterations divided
1790 // by the number of threads in the team, decreasing to 1. For a chunk_size
1791 // with value k (greater than 1), the size of each chunk is determined in the
1792 // same way, with the restriction that the chunks do not contain fewer than k
1793 // iterations (except for the last chunk to be assigned, which may have fewer
1794 // than k iterations).
1795 //
1796 // When schedule(auto) is specified, the decision regarding scheduling is
1797 // delegated to the compiler and/or runtime system. The programmer gives the
1798 // implementation the freedom to choose any possible mapping of iterations to
1799 // threads in the team.
1800 //
1801 // When schedule(runtime) is specified, the decision regarding scheduling is
1802 // deferred until run time, and the schedule and chunk size are taken from the
1803 // run-sched-var ICV. If the ICV is set to auto, the schedule is
1804 // implementation defined
1805 //
1806 // while(__kmpc_dispatch_next(&LB, &UB)) {
1807 // idx = LB;
1808 // while (idx <= UB) { BODY; ++idx;
1809 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1810 // } // inner loop
1811 // }
1812 //
1813 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1814 // When schedule(static, chunk_size) is specified, iterations are divided into
1815 // chunks of size chunk_size, and the chunks are assigned to the threads in
1816 // the team in a round-robin fashion in the order of the thread number.
1817 //
1818 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1819 // while (idx <= UB) { BODY; ++idx; } // inner loop
1820 // LB = LB + ST;
1821 // UB = UB + ST;
1822 // }
1823 //
1824
1825 const Expr *IVExpr = S.getIterationVariable();
1826 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1827 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1828
1829 if (DynamicOrOrdered) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001830 auto DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1831 llvm::Value *LBVal = DispatchBounds.first;
1832 llvm::Value *UBVal = DispatchBounds.second;
1833 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1834 LoopArgs.Chunk};
Alexey Bataev9ebd7422016-05-10 09:57:36 +00001835 RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001836 IVSigned, Ordered, DipatchRTInputValues);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001837 } else {
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00001838 CGOpenMPRuntime::StaticRTInput StaticInit(
1839 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1840 LoopArgs.ST, LoopArgs.Chunk);
1841 RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
1842 ScheduleKind, StaticInit);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001843 }
1844
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001845 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1846 const unsigned IVSize,
1847 const bool IVSigned) {
1848 if (Ordered) {
1849 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1850 IVSigned);
1851 }
1852 };
1853
1854 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1855 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1856 OuterLoopArgs.IncExpr = S.getInc();
1857 OuterLoopArgs.Init = S.getInit();
1858 OuterLoopArgs.Cond = S.getCond();
1859 OuterLoopArgs.NextLB = S.getNextLowerBound();
1860 OuterLoopArgs.NextUB = S.getNextUpperBound();
1861 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1862 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001863}
1864
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001865static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
1866 const unsigned IVSize, const bool IVSigned) {}
1867
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001868void CodeGenFunction::EmitOMPDistributeOuterLoop(
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001869 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1870 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1871 const CodeGenLoopTy &CodeGenLoopContent) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001872
1873 auto &RT = CGM.getOpenMPRuntime();
1874
1875 // Emit outer loop.
1876 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1877 // dynamic
1878 //
1879
1880 const Expr *IVExpr = S.getIterationVariable();
1881 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1882 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1883
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00001884 CGOpenMPRuntime::StaticRTInput StaticInit(
1885 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
1886 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
1887 RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00001888
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001889 // for combined 'distribute' and 'for' the increment expression of distribute
1890 // is store in DistInc. For 'distribute' alone, it is in Inc.
1891 Expr *IncExpr;
1892 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
1893 IncExpr = S.getDistInc();
1894 else
1895 IncExpr = S.getInc();
1896
1897 // this routine is shared by 'omp distribute parallel for' and
1898 // 'omp distribute': select the right EUB expression depending on the
1899 // directive
1900 OMPLoopArguments OuterLoopArgs;
1901 OuterLoopArgs.LB = LoopArgs.LB;
1902 OuterLoopArgs.UB = LoopArgs.UB;
1903 OuterLoopArgs.ST = LoopArgs.ST;
1904 OuterLoopArgs.IL = LoopArgs.IL;
1905 OuterLoopArgs.Chunk = LoopArgs.Chunk;
1906 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1907 ? S.getCombinedEnsureUpperBound()
1908 : S.getEnsureUpperBound();
1909 OuterLoopArgs.IncExpr = IncExpr;
1910 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1911 ? S.getCombinedInit()
1912 : S.getInit();
1913 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1914 ? S.getCombinedCond()
1915 : S.getCond();
1916 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1917 ? S.getCombinedNextLowerBound()
1918 : S.getNextLowerBound();
1919 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1920 ? S.getCombinedNextUpperBound()
1921 : S.getNextUpperBound();
1922
1923 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
1924 LoopScope, OuterLoopArgs, CodeGenLoopContent,
1925 emitEmptyOrdered);
1926}
1927
1928/// Emit a helper variable and return corresponding lvalue.
1929static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1930 const DeclRefExpr *Helper) {
1931 auto VDecl = cast<VarDecl>(Helper->getDecl());
1932 CGF.EmitVarDecl(*VDecl);
1933 return CGF.EmitLValue(Helper);
1934}
1935
1936static std::pair<LValue, LValue>
1937emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
1938 const OMPExecutableDirective &S) {
1939 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
1940 LValue LB =
1941 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
1942 LValue UB =
1943 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
1944
1945 // When composing 'distribute' with 'for' (e.g. as in 'distribute
1946 // parallel for') we need to use the 'distribute'
1947 // chunk lower and upper bounds rather than the whole loop iteration
1948 // space. These are parameters to the outlined function for 'parallel'
1949 // and we copy the bounds of the previous schedule into the
1950 // the current ones.
1951 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
1952 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
1953 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(PrevLB, SourceLocation());
1954 PrevLBVal = CGF.EmitScalarConversion(
1955 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
1956 LS.getIterationVariable()->getType(), SourceLocation());
1957 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(PrevUB, SourceLocation());
1958 PrevUBVal = CGF.EmitScalarConversion(
1959 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
1960 LS.getIterationVariable()->getType(), SourceLocation());
1961
1962 CGF.EmitStoreOfScalar(PrevLBVal, LB);
1963 CGF.EmitStoreOfScalar(PrevUBVal, UB);
1964
1965 return {LB, UB};
1966}
1967
1968/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
1969/// we need to use the LB and UB expressions generated by the worksharing
1970/// code generation support, whereas in non combined situations we would
1971/// just emit 0 and the LastIteration expression
1972/// This function is necessary due to the difference of the LB and UB
1973/// types for the RT emission routines for 'for_static_init' and
1974/// 'for_dispatch_init'
1975static std::pair<llvm::Value *, llvm::Value *>
1976emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
1977 const OMPExecutableDirective &S,
1978 Address LB, Address UB) {
1979 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
1980 const Expr *IVExpr = LS.getIterationVariable();
1981 // when implementing a dynamic schedule for a 'for' combined with a
1982 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
1983 // is not normalized as each team only executes its own assigned
1984 // distribute chunk
1985 QualType IteratorTy = IVExpr->getType();
1986 llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy,
1987 SourceLocation());
1988 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy,
1989 SourceLocation());
1990 return {LBVal, UBVal};
Hans Wennborged129ae2017-04-27 17:02:25 +00001991}
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001992
1993static void emitDistributeParallelForDistributeInnerBoundParams(
1994 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1995 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
1996 const auto &Dir = cast<OMPLoopDirective>(S);
1997 LValue LB =
1998 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
1999 auto LBCast = CGF.Builder.CreateIntCast(
2000 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2001 CapturedVars.push_back(LBCast);
2002 LValue UB =
2003 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2004
2005 auto UBCast = CGF.Builder.CreateIntCast(
2006 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2007 CapturedVars.push_back(UBCast);
Hans Wennborged129ae2017-04-27 17:02:25 +00002008}
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002009
2010static void
2011emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
2012 const OMPLoopDirective &S,
2013 CodeGenFunction::JumpDest LoopExit) {
2014 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2015 PrePostActionTy &) {
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00002016 bool HasCancel = false;
2017 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2018 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2019 HasCancel = D->hasCancel();
2020 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2021 HasCancel = D->hasCancel();
Alexey Bataev16e79882017-11-22 21:12:03 +00002022 else if (const auto *D =
2023 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2024 HasCancel = D->hasCancel();
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00002025 }
2026 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
2027 HasCancel);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002028 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
2029 emitDistributeParallelForInnerBounds,
2030 emitDistributeParallelForDispatchBounds);
2031 };
2032
2033 emitCommonOMPParallelDirective(
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00002034 CGF, S,
2035 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2036 CGInlinedWorksharingLoop,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002037 emitDistributeParallelForDistributeInnerBoundParams);
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002038}
2039
Carlo Bertolli9925f152016-06-27 14:55:37 +00002040void CodeGenFunction::EmitOMPDistributeParallelForDirective(
2041 const OMPDistributeParallelForDirective &S) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002042 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2043 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2044 S.getDistInc());
2045 };
Carlo Bertolli9925f152016-06-27 14:55:37 +00002046 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev10a54312017-11-27 16:54:08 +00002047 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
Carlo Bertolli9925f152016-06-27 14:55:37 +00002048}
2049
Kelvin Li4a39add2016-07-05 05:00:15 +00002050void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
2051 const OMPDistributeParallelForSimdDirective &S) {
Alexey Bataev0b49f9e2017-11-27 19:38:58 +00002052 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2053 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2054 S.getDistInc());
2055 };
Kelvin Li4a39add2016-07-05 05:00:15 +00002056 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev0b49f9e2017-11-27 19:38:58 +00002057 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
Kelvin Li4a39add2016-07-05 05:00:15 +00002058}
Kelvin Li787f3fc2016-07-06 04:45:38 +00002059
2060void CodeGenFunction::EmitOMPDistributeSimdDirective(
2061 const OMPDistributeSimdDirective &S) {
Alexey Bataev617db5f2017-12-04 15:38:33 +00002062 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2063 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
2064 };
Kelvin Li787f3fc2016-07-06 04:45:38 +00002065 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev617db5f2017-12-04 15:38:33 +00002066 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
Kelvin Li787f3fc2016-07-06 04:45:38 +00002067}
2068
Alexey Bataevf8365372017-11-17 17:57:25 +00002069void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
2070 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2071 // Emit SPMD target parallel for region as a standalone region.
2072 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2073 emitOMPSimdRegion(CGF, S, Action);
2074 };
2075 llvm::Function *Fn;
2076 llvm::Constant *Addr;
2077 // Emit target region as a standalone region.
2078 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2079 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2080 assert(Fn && Addr && "Target device function emission failed.");
2081}
2082
Kelvin Li986330c2016-07-20 22:57:10 +00002083void CodeGenFunction::EmitOMPTargetSimdDirective(
2084 const OMPTargetSimdDirective &S) {
Alexey Bataevf8365372017-11-17 17:57:25 +00002085 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2086 emitOMPSimdRegion(CGF, S, Action);
2087 };
2088 emitCommonOMPTargetDirective(*this, S, CodeGen);
Kelvin Li986330c2016-07-20 22:57:10 +00002089}
2090
Kelvin Li4e325f72016-10-25 12:50:55 +00002091void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
2092 const OMPTeamsDistributeSimdDirective &S) {
2093 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2094 CGM.getOpenMPRuntime().emitInlinedDirective(
2095 *this, OMPD_teams_distribute_simd,
2096 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2097 OMPLoopScope PreInitScope(CGF, S);
2098 CGF.EmitStmt(
2099 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2100 });
2101}
2102
Kelvin Li83c451e2016-12-25 04:52:54 +00002103void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
2104 const OMPTargetTeamsDistributeDirective &S) {
Alexey Bataev931e19b2017-10-02 16:32:39 +00002105 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Kelvin Li26fd21a2016-12-28 17:57:07 +00002106 CGM.getOpenMPRuntime().emitInlinedDirective(
2107 *this, OMPD_target_teams_distribute,
Kelvin Li83c451e2016-12-25 04:52:54 +00002108 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Kelvin Li26fd21a2016-12-28 17:57:07 +00002109 CGF.EmitStmt(
2110 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
Kelvin Li83c451e2016-12-25 04:52:54 +00002111 });
2112}
2113
Kelvin Li80e8f562016-12-29 22:16:30 +00002114void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
2115 const OMPTargetTeamsDistributeParallelForDirective &S) {
Alexey Bataev931e19b2017-10-02 16:32:39 +00002116 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Kelvin Li80e8f562016-12-29 22:16:30 +00002117 CGM.getOpenMPRuntime().emitInlinedDirective(
2118 *this, OMPD_target_teams_distribute_parallel_for,
2119 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2120 CGF.EmitStmt(
2121 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2122 });
2123}
2124
Kelvin Li1851df52017-01-03 05:23:48 +00002125void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
2126 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
Alexey Bataev931e19b2017-10-02 16:32:39 +00002127 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Kelvin Li1851df52017-01-03 05:23:48 +00002128 CGM.getOpenMPRuntime().emitInlinedDirective(
2129 *this, OMPD_target_teams_distribute_parallel_for_simd,
2130 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2131 CGF.EmitStmt(
2132 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2133 });
2134}
2135
Kelvin Lida681182017-01-10 18:08:18 +00002136void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
2137 const OMPTargetTeamsDistributeSimdDirective &S) {
Alexey Bataev931e19b2017-10-02 16:32:39 +00002138 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Kelvin Lida681182017-01-10 18:08:18 +00002139 CGM.getOpenMPRuntime().emitInlinedDirective(
2140 *this, OMPD_target_teams_distribute_simd,
2141 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2142 CGF.EmitStmt(
2143 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2144 });
2145}
2146
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002147namespace {
2148 struct ScheduleKindModifiersTy {
2149 OpenMPScheduleClauseKind Kind;
2150 OpenMPScheduleClauseModifier M1;
2151 OpenMPScheduleClauseModifier M2;
2152 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2153 OpenMPScheduleClauseModifier M1,
2154 OpenMPScheduleClauseModifier M2)
2155 : Kind(Kind), M1(M1), M2(M2) {}
2156 };
2157} // namespace
2158
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002159bool CodeGenFunction::EmitOMPWorksharingLoop(
2160 const OMPLoopDirective &S, Expr *EUB,
2161 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2162 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
Alexander Musmanc6388682014-12-15 07:07:06 +00002163 // Emit the loop iteration variable.
2164 auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2165 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2166 EmitVarDecl(*IVDecl);
2167
2168 // Emit the iterations count variable.
2169 // If it is not a variable, Sema decided to calculate iterations count on each
2170 // iteration (e.g., it is foldable into a constant).
2171 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2172 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2173 // Emit calculation of the iterations count.
2174 EmitIgnoredExpr(S.getCalcLastIteration());
2175 }
2176
2177 auto &RT = CGM.getOpenMPRuntime();
2178
Alexey Bataev38e89532015-04-16 04:54:05 +00002179 bool HasLastprivateClause;
Alexander Musmanc6388682014-12-15 07:07:06 +00002180 // Check pre-condition.
2181 {
Alexey Bataev5a3af132016-03-29 08:58:54 +00002182 OMPLoopScope PreInitScope(*this, S);
Alexander Musmanc6388682014-12-15 07:07:06 +00002183 // Skip the entire loop if we don't meet the precondition.
Alexey Bataev62dbb972015-04-22 11:59:37 +00002184 // If the condition constant folds and can be elided, avoid emitting the
2185 // whole loop.
2186 bool CondConstant;
2187 llvm::BasicBlock *ContBlock = nullptr;
2188 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2189 if (!CondConstant)
2190 return false;
2191 } else {
Alexey Bataev62dbb972015-04-22 11:59:37 +00002192 auto *ThenBlock = createBasicBlock("omp.precond.then");
2193 ContBlock = createBasicBlock("omp.precond.end");
2194 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
Justin Bogner66242d62015-04-23 23:06:47 +00002195 getProfileCount(&S));
Alexey Bataev62dbb972015-04-22 11:59:37 +00002196 EmitBlock(ThenBlock);
Justin Bogner66242d62015-04-23 23:06:47 +00002197 incrementProfileCounter(&S);
Alexey Bataev62dbb972015-04-22 11:59:37 +00002198 }
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00002199
Alexey Bataev8b427062016-05-25 12:36:08 +00002200 bool Ordered = false;
2201 if (auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2202 if (OrderedClause->getNumForLoops())
2203 RT.emitDoacrossInit(*this, S);
2204 else
2205 Ordered = true;
2206 }
2207
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002208 llvm::DenseSet<const Expr *> EmittedFinals;
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00002209 emitAlignedClause(*this, S);
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00002210 bool HasLinears = EmitOMPLinearClauseInit(S);
Alexey Bataevef549a82016-03-09 09:49:09 +00002211 // Emit helper vars inits.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002212
2213 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2214 LValue LB = Bounds.first;
2215 LValue UB = Bounds.second;
Alexey Bataevef549a82016-03-09 09:49:09 +00002216 LValue ST =
2217 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2218 LValue IL =
2219 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2220
Alexander Musmanc6388682014-12-15 07:07:06 +00002221 // Emit 'then' code.
2222 {
Alexander Musmanc6388682014-12-15 07:07:06 +00002223 OMPPrivateScope LoopScope(*this);
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00002224 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
Alexey Bataev69c62a92015-04-15 04:52:20 +00002225 // Emit implicit barrier to synchronize threads and avoid data races on
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00002226 // initialization of firstprivate variables and post-update of
2227 // lastprivate variables.
Alexey Bataev25e5b442015-09-15 12:52:43 +00002228 CGM.getOpenMPRuntime().emitBarrierCall(
2229 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2230 /*ForceSimpleCall=*/true);
Alexey Bataev69c62a92015-04-15 04:52:20 +00002231 }
Alexey Bataev50a64582015-04-22 12:24:45 +00002232 EmitOMPPrivateClause(S, LoopScope);
Alexey Bataev38e89532015-04-16 04:54:05 +00002233 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
Alexey Bataev7ebe5fd2015-04-22 13:43:03 +00002234 EmitOMPReductionClauseInit(S, LoopScope);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002235 EmitOMPPrivateLoopCounters(S, LoopScope);
2236 EmitOMPLinearClause(S, LoopScope);
Alexander Musman7931b982015-03-16 07:14:41 +00002237 (void)LoopScope.Privatize();
Alexander Musmanc6388682014-12-15 07:07:06 +00002238
2239 // Detect the loop schedule kind and chunk.
Alexey Bataev3392d762016-02-16 11:18:12 +00002240 llvm::Value *Chunk = nullptr;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002241 OpenMPScheduleTy ScheduleKind;
Alexey Bataev3392d762016-02-16 11:18:12 +00002242 if (auto *C = S.getSingleClause<OMPScheduleClause>()) {
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002243 ScheduleKind.Schedule = C->getScheduleKind();
2244 ScheduleKind.M1 = C->getFirstScheduleModifier();
2245 ScheduleKind.M2 = C->getSecondScheduleModifier();
Alexey Bataev3392d762016-02-16 11:18:12 +00002246 if (const auto *Ch = C->getChunkSize()) {
2247 Chunk = EmitScalarExpr(Ch);
2248 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2249 S.getIterationVariable()->getType(),
2250 S.getLocStart());
2251 }
2252 }
Alexander Musmanc6388682014-12-15 07:07:06 +00002253 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2254 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002255 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2256 // If the static schedule kind is specified or if the ordered clause is
2257 // specified, and if no monotonic modifier is specified, the effect will
2258 // be as if the monotonic modifier was specified.
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002259 if (RT.isStaticNonchunked(ScheduleKind.Schedule,
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002260 /* Chunked */ Chunk != nullptr) &&
2261 !Ordered) {
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002262 if (isOpenMPSimdDirective(S.getDirectiveKind()))
2263 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
Alexander Musmanc6388682014-12-15 07:07:06 +00002264 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2265 // When no chunk_size is specified, the iteration space is divided into
2266 // chunks that are approximately equal in size, and at most one chunk is
2267 // distributed to each thread. Note that the size of the chunks is
2268 // unspecified in this case.
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002269 CGOpenMPRuntime::StaticRTInput StaticInit(
2270 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2271 UB.getAddress(), ST.getAddress());
2272 RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
2273 ScheduleKind, StaticInit);
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002274 auto LoopExit =
2275 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
Alexander Musmanc6388682014-12-15 07:07:06 +00002276 // UB = min(UB, GlobalUB);
2277 EmitIgnoredExpr(S.getEnsureUpperBound());
2278 // IV = LB;
2279 EmitIgnoredExpr(S.getInit());
2280 // while (idx <= UB) { BODY; ++idx; }
Alexey Bataevae05c292015-06-16 11:59:36 +00002281 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2282 S.getInc(),
Alexey Bataev0f34da12015-07-02 04:17:07 +00002283 [&S, LoopExit](CodeGenFunction &CGF) {
2284 CGF.EmitOMPLoopBody(S, LoopExit);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002285 CGF.EmitStopPoint(&S);
Alexey Bataev98eb6e32015-04-22 11:15:40 +00002286 },
2287 [](CodeGenFunction &) {});
Alexey Bataev0f34da12015-07-02 04:17:07 +00002288 EmitBlock(LoopExit.getBlock());
Alexander Musmanc6388682014-12-15 07:07:06 +00002289 // Tell the runtime we are done.
Alexey Bataev957d8562016-11-17 15:12:05 +00002290 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
Alexey Bataevf43f7142017-09-06 16:17:35 +00002291 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2292 S.getDirectiveKind());
Alexey Bataev957d8562016-11-17 15:12:05 +00002293 };
2294 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002295 } else {
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002296 const bool IsMonotonic =
2297 Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2298 ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2299 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2300 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002301 // Emit the outer loop, which requests its work chunk [LB..UB] from
2302 // runtime and runs the inner loop to process it.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002303 const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2304 ST.getAddress(), IL.getAddress(),
2305 Chunk, EUB);
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002306 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002307 LoopArguments, CGDispatchBounds);
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002308 }
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002309 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2310 EmitOMPSimdFinal(S,
2311 [&](CodeGenFunction &CGF) -> llvm::Value * {
2312 return CGF.Builder.CreateIsNotNull(
2313 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2314 });
2315 }
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00002316 EmitOMPReductionClauseFinal(
2317 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2318 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2319 : /*Parallel only*/ OMPD_parallel);
Alexey Bataev61205072016-03-02 04:57:40 +00002320 // Emit post-update of the reduction variables if IsLastIter != 0.
2321 emitPostUpdateForReductionClause(
2322 *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2323 return CGF.Builder.CreateIsNotNull(
2324 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2325 });
Alexey Bataev38e89532015-04-16 04:54:05 +00002326 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2327 if (HasLastprivateClause)
2328 EmitOMPLastprivateClauseFinal(
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002329 S, isOpenMPSimdDirective(S.getDirectiveKind()),
2330 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
Alexander Musmanc6388682014-12-15 07:07:06 +00002331 }
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002332 EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
Alexey Bataevef549a82016-03-09 09:49:09 +00002333 return CGF.Builder.CreateIsNotNull(
2334 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2335 });
Alexander Musmanc6388682014-12-15 07:07:06 +00002336 // We're now done with the loop, so jump to the continuation block.
Alexey Bataev62dbb972015-04-22 11:59:37 +00002337 if (ContBlock) {
2338 EmitBranch(ContBlock);
2339 EmitBlock(ContBlock, true);
2340 }
Alexander Musmanc6388682014-12-15 07:07:06 +00002341 }
Alexey Bataev38e89532015-04-16 04:54:05 +00002342 return HasLastprivateClause;
Alexander Musmanc6388682014-12-15 07:07:06 +00002343}
2344
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002345/// The following two functions generate expressions for the loop lower
2346/// and upper bounds in case of static and dynamic (dispatch) schedule
2347/// of the associated 'for' or 'distribute' loop.
2348static std::pair<LValue, LValue>
2349emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
2350 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2351 LValue LB =
2352 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2353 LValue UB =
2354 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2355 return {LB, UB};
2356}
2357
2358/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2359/// consider the lower and upper bound expressions generated by the
2360/// worksharing loop support, but we use 0 and the iteration space size as
2361/// constants
2362static std::pair<llvm::Value *, llvm::Value *>
2363emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2364 Address LB, Address UB) {
2365 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2366 const Expr *IVExpr = LS.getIterationVariable();
2367 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2368 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2369 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2370 return {LBVal, UBVal};
2371}
2372
Alexander Musmanc6388682014-12-15 07:07:06 +00002373void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
Alexey Bataev38e89532015-04-16 04:54:05 +00002374 bool HasLastprivates = false;
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002375 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2376 PrePostActionTy &) {
Alexey Bataev957d8562016-11-17 15:12:05 +00002377 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002378 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2379 emitForLoopBounds,
2380 emitDispatchForLoopBounds);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002381 };
Alexey Bataev3392d762016-02-16 11:18:12 +00002382 {
Alexey Bataev4ba78a42016-04-27 07:56:03 +00002383 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev3392d762016-02-16 11:18:12 +00002384 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2385 S.hasCancel());
2386 }
Alexander Musmanc6388682014-12-15 07:07:06 +00002387
2388 // Emit an implicit barrier at the end.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002389 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
Alexey Bataevf2685682015-03-30 04:30:22 +00002390 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2391 }
Alexey Bataevf29276e2014-06-18 04:14:57 +00002392}
Alexey Bataevd3f8dd22014-06-25 11:44:49 +00002393
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00002394void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00002395 bool HasLastprivates = false;
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002396 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2397 PrePostActionTy &) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002398 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2399 emitForLoopBounds,
2400 emitDispatchForLoopBounds);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002401 };
Alexey Bataev3392d762016-02-16 11:18:12 +00002402 {
Alexey Bataev4ba78a42016-04-27 07:56:03 +00002403 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev3392d762016-02-16 11:18:12 +00002404 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2405 }
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00002406
2407 // Emit an implicit barrier at the end.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002408 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00002409 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2410 }
Alexander Musmanf82886e2014-09-18 05:12:34 +00002411}
2412
Alexey Bataev2df54a02015-03-12 08:53:29 +00002413static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2414 const Twine &Name,
2415 llvm::Value *Init = nullptr) {
John McCall7f416cc2015-09-08 08:05:57 +00002416 auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
Alexey Bataev2df54a02015-03-12 08:53:29 +00002417 if (Init)
Akira Hatanaka642f7992016-10-18 19:05:41 +00002418 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
Alexey Bataev2df54a02015-03-12 08:53:29 +00002419 return LVal;
Alexey Bataevd3f8dd22014-06-25 11:44:49 +00002420}
2421
Alexey Bataev3392d762016-02-16 11:18:12 +00002422void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
Alexey Bataev2df54a02015-03-12 08:53:29 +00002423 auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
2424 auto *CS = dyn_cast<CompoundStmt>(Stmt);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002425 bool HasLastprivates = false;
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002426 auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF,
2427 PrePostActionTy &) {
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002428 auto &C = CGF.CGM.getContext();
2429 auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2430 // Emit helper vars inits.
2431 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2432 CGF.Builder.getInt32(0));
2433 auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2434 : CGF.Builder.getInt32(0);
2435 LValue UB =
2436 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2437 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2438 CGF.Builder.getInt32(1));
2439 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2440 CGF.Builder.getInt32(0));
2441 // Loop counter.
2442 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2443 OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2444 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2445 OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2446 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2447 // Generate condition for loop.
2448 BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
Adam Nemet484aa452017-03-27 19:17:25 +00002449 OK_Ordinary, S.getLocStart(), FPOptions());
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002450 // Increment for loop counter.
2451 UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2452 S.getLocStart());
2453 auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) {
2454 // Iterate through all sections and emit a switch construct:
2455 // switch (IV) {
2456 // case 0:
2457 // <SectionStmt[0]>;
2458 // break;
2459 // ...
2460 // case <NumSection> - 1:
2461 // <SectionStmt[<NumSection> - 1]>;
2462 // break;
2463 // }
2464 // .omp.sections.exit:
2465 auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2466 auto *SwitchStmt = CGF.Builder.CreateSwitch(
2467 CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
2468 CS == nullptr ? 1 : CS->size());
2469 if (CS) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002470 unsigned CaseNumber = 0;
Benjamin Kramer642f1732015-07-02 21:03:14 +00002471 for (auto *SubStmt : CS->children()) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002472 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2473 CGF.EmitBlock(CaseBB);
2474 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
Benjamin Kramer642f1732015-07-02 21:03:14 +00002475 CGF.EmitStmt(SubStmt);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002476 CGF.EmitBranch(ExitBB);
Benjamin Kramer642f1732015-07-02 21:03:14 +00002477 ++CaseNumber;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002478 }
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002479 } else {
2480 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2481 CGF.EmitBlock(CaseBB);
2482 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2483 CGF.EmitStmt(Stmt);
2484 CGF.EmitBranch(ExitBB);
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002485 }
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002486 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
Alexey Bataev2df54a02015-03-12 08:53:29 +00002487 };
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002488
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002489 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2490 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
Alexey Bataev9efc03b2015-04-27 04:34:03 +00002491 // Emit implicit barrier to synchronize threads and avoid data races on
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00002492 // initialization of firstprivate variables and post-update of lastprivate
2493 // variables.
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002494 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2495 CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2496 /*ForceSimpleCall=*/true);
Alexey Bataev9efc03b2015-04-27 04:34:03 +00002497 }
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002498 CGF.EmitOMPPrivateClause(S, LoopScope);
2499 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2500 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2501 (void)LoopScope.Privatize();
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002502
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002503 // Emit static non-chunked loop.
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002504 OpenMPScheduleTy ScheduleKind;
2505 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002506 CGOpenMPRuntime::StaticRTInput StaticInit(
2507 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2508 LB.getAddress(), UB.getAddress(), ST.getAddress());
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002509 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002510 CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002511 // UB = min(UB, GlobalUB);
2512 auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2513 auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2514 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2515 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2516 // IV = LB;
2517 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2518 // while (idx <= UB) { BODY; ++idx; }
2519 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2520 [](CodeGenFunction &) {});
2521 // Tell the runtime we are done.
Alexey Bataev957d8562016-11-17 15:12:05 +00002522 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
Alexey Bataevf43f7142017-09-06 16:17:35 +00002523 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2524 S.getDirectiveKind());
Alexey Bataev957d8562016-11-17 15:12:05 +00002525 };
2526 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00002527 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
Alexey Bataev61205072016-03-02 04:57:40 +00002528 // Emit post-update of the reduction variables if IsLastIter != 0.
2529 emitPostUpdateForReductionClause(
2530 CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2531 return CGF.Builder.CreateIsNotNull(
2532 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2533 });
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002534
2535 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2536 if (HasLastprivates)
2537 CGF.EmitOMPLastprivateClauseFinal(
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002538 S, /*NoFinals=*/false,
2539 CGF.Builder.CreateIsNotNull(
2540 CGF.EmitLoadOfScalar(IL, S.getLocStart())));
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002541 };
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002542
2543 bool HasCancel = false;
2544 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2545 HasCancel = OSD->hasCancel();
2546 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2547 HasCancel = OPSD->hasCancel();
Alexey Bataev957d8562016-11-17 15:12:05 +00002548 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002549 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2550 HasCancel);
2551 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2552 // clause. Otherwise the barrier will be generated by the codegen for the
2553 // directive.
2554 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002555 // Emit implicit barrier to synchronize threads and avoid data races on
2556 // initialization of firstprivate variables.
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002557 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2558 OMPD_unknown);
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002559 }
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002560}
Alexey Bataev2df54a02015-03-12 08:53:29 +00002561
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002562void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
Alexey Bataev3392d762016-02-16 11:18:12 +00002563 {
Alexey Bataev4ba78a42016-04-27 07:56:03 +00002564 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev3392d762016-02-16 11:18:12 +00002565 EmitSections(S);
2566 }
Alexey Bataev2df54a02015-03-12 08:53:29 +00002567 // Emit an implicit barrier at the end.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002568 if (!S.getSingleClause<OMPNowaitClause>()) {
Alexey Bataev3392d762016-02-16 11:18:12 +00002569 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2570 OMPD_sections);
Alexey Bataevf2685682015-03-30 04:30:22 +00002571 }
Alexey Bataev2df54a02015-03-12 08:53:29 +00002572}
2573
2574void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002575 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002576 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002577 };
Alexey Bataev4ba78a42016-04-27 07:56:03 +00002578 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev25e5b442015-09-15 12:52:43 +00002579 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2580 S.hasCancel());
Alexey Bataev1e0498a2014-06-26 08:21:58 +00002581}
2582
Alexey Bataev6956e2e2015-02-05 06:35:41 +00002583void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
Alexey Bataeva63048e2015-03-23 06:18:07 +00002584 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
Alexey Bataev420d45b2015-04-14 05:11:24 +00002585 llvm::SmallVector<const Expr *, 8> DestExprs;
Alexey Bataeva63048e2015-03-23 06:18:07 +00002586 llvm::SmallVector<const Expr *, 8> SrcExprs;
Alexey Bataeva63048e2015-03-23 06:18:07 +00002587 llvm::SmallVector<const Expr *, 8> AssignmentOps;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002588 // Check if there are any 'copyprivate' clauses associated with this
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00002589 // 'single' construct.
Alexey Bataeva63048e2015-03-23 06:18:07 +00002590 // Build a list of copyprivate variables along with helper expressions
2591 // (<source>, <destination>, <destination>=<source> expressions)
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002592 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
Alexey Bataeva63048e2015-03-23 06:18:07 +00002593 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
Alexey Bataev420d45b2015-04-14 05:11:24 +00002594 DestExprs.append(C->destination_exprs().begin(),
2595 C->destination_exprs().end());
Alexey Bataeva63048e2015-03-23 06:18:07 +00002596 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
Alexey Bataeva63048e2015-03-23 06:18:07 +00002597 AssignmentOps.append(C->assignment_ops().begin(),
2598 C->assignment_ops().end());
2599 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002600 // Emit code for 'single' region along with 'copyprivate' clauses
2601 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2602 Action.Enter(CGF);
2603 OMPPrivateScope SingleScope(CGF);
2604 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2605 CGF.EmitOMPPrivateClause(S, SingleScope);
2606 (void)SingleScope.Privatize();
2607 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2608 };
Alexey Bataev3392d762016-02-16 11:18:12 +00002609 {
Alexey Bataev4ba78a42016-04-27 07:56:03 +00002610 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev3392d762016-02-16 11:18:12 +00002611 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
2612 CopyprivateVars, DestExprs,
2613 SrcExprs, AssignmentOps);
2614 }
2615 // Emit an implicit barrier at the end (to avoid data race on firstprivate
2616 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
Alexey Bataev417089f2016-02-17 13:19:37 +00002617 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
Alexey Bataev5521d782015-04-24 04:21:15 +00002618 CGM.getOpenMPRuntime().emitBarrierCall(
2619 *this, S.getLocStart(),
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002620 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
Alexey Bataevf2685682015-03-30 04:30:22 +00002621 }
Alexey Bataevd1e40fb2014-06-26 12:05:45 +00002622}
2623
Alexey Bataev8d690652014-12-04 07:23:53 +00002624void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002625 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2626 Action.Enter(CGF);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002627 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002628 };
Alexey Bataev4ba78a42016-04-27 07:56:03 +00002629 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002630 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
Alexander Musman80c22892014-07-17 08:54:58 +00002631}
2632
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00002633void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002634 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2635 Action.Enter(CGF);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002636 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002637 };
Alexey Bataevfc57d162015-12-15 10:55:09 +00002638 Expr *Hint = nullptr;
2639 if (auto *HintClause = S.getSingleClause<OMPHintClause>())
2640 Hint = HintClause->getHint();
Alexey Bataev4ba78a42016-04-27 07:56:03 +00002641 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataevfc57d162015-12-15 10:55:09 +00002642 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2643 S.getDirectiveName().getAsString(),
2644 CodeGen, S.getLocStart(), Hint);
Alexander Musmand9ed09f2014-07-21 09:42:05 +00002645}
2646
Alexey Bataev671605e2015-04-13 05:28:11 +00002647void CodeGenFunction::EmitOMPParallelForDirective(
2648 const OMPParallelForDirective &S) {
2649 // Emit directive as a combined directive that consists of two implicit
2650 // directives: 'parallel' with 'for' directive.
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002651 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev957d8562016-11-17 15:12:05 +00002652 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002653 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2654 emitDispatchForLoopBounds);
Alexey Bataev671605e2015-04-13 05:28:11 +00002655 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002656 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2657 emitEmptyBoundParameters);
Alexey Bataev4acb8592014-07-07 13:01:15 +00002658}
2659
Alexander Musmane4e893b2014-09-23 09:33:00 +00002660void CodeGenFunction::EmitOMPParallelForSimdDirective(
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00002661 const OMPParallelForSimdDirective &S) {
2662 // Emit directive as a combined directive that consists of two implicit
2663 // directives: 'parallel' with 'for' directive.
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002664 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002665 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2666 emitDispatchForLoopBounds);
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00002667 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002668 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2669 emitEmptyBoundParameters);
Alexander Musmane4e893b2014-09-23 09:33:00 +00002670}
2671
Alexey Bataev84d0b3e2014-07-08 08:12:03 +00002672void CodeGenFunction::EmitOMPParallelSectionsDirective(
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002673 const OMPParallelSectionsDirective &S) {
2674 // Emit directive as a combined directive that consists of two implicit
2675 // directives: 'parallel' with 'sections' directive.
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002676 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2677 CGF.EmitSections(S);
2678 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002679 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2680 emitEmptyBoundParameters);
Alexey Bataev84d0b3e2014-07-08 08:12:03 +00002681}
2682
Alexey Bataev7292c292016-04-25 12:22:29 +00002683void CodeGenFunction::EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
2684 const RegionCodeGenTy &BodyGen,
2685 const TaskGenTy &TaskGen,
Alexey Bataev24b5bae2016-04-28 09:23:51 +00002686 OMPTaskDataTy &Data) {
Alexey Bataev62b63b12015-03-10 07:28:44 +00002687 // Emit outlined function for task construct.
2688 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
Alexey Bataev62b63b12015-03-10 07:28:44 +00002689 auto *I = CS->getCapturedDecl()->param_begin();
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002690 auto *PartId = std::next(I);
Alexey Bataev48591dd2016-04-20 04:01:36 +00002691 auto *TaskT = std::next(I, 4);
Alexey Bataev24b5bae2016-04-28 09:23:51 +00002692 // Check if the task is final
2693 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2694 // If the condition constant folds and can be elided, try to avoid emitting
2695 // the condition and the dead arm of the if/else.
2696 auto *Cond = Clause->getCondition();
2697 bool CondConstant;
2698 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2699 Data.Final.setInt(CondConstant);
2700 else
2701 Data.Final.setPointer(EvaluateExprAsBool(Cond));
2702 } else {
2703 // By default the task is not final.
2704 Data.Final.setInt(/*IntVal=*/false);
2705 }
Alexey Bataev1e1e2862016-05-10 12:21:02 +00002706 // Check if the task has 'priority' clause.
2707 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
Alexey Bataev1e1e2862016-05-10 12:21:02 +00002708 auto *Prio = Clause->getPriority();
Alexey Bataev5140e742016-07-19 04:21:09 +00002709 Data.Priority.setInt(/*IntVal=*/true);
Alexey Bataevad537bb2016-05-30 09:06:50 +00002710 Data.Priority.setPointer(EmitScalarConversion(
2711 EmitScalarExpr(Prio), Prio->getType(),
2712 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2713 Prio->getExprLoc()));
Alexey Bataev1e1e2862016-05-10 12:21:02 +00002714 }
Alexey Bataev62b63b12015-03-10 07:28:44 +00002715 // The first function argument for tasks is a thread id, the second one is a
2716 // part id (0 for tied tasks, >=0 for untied task).
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002717 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2718 // Get list of private variables.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002719 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002720 auto IRef = C->varlist_begin();
2721 for (auto *IInit : C->private_copies()) {
2722 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2723 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataev7292c292016-04-25 12:22:29 +00002724 Data.PrivateVars.push_back(*IRef);
2725 Data.PrivateCopies.push_back(IInit);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002726 }
2727 ++IRef;
2728 }
2729 }
2730 EmittedAsPrivate.clear();
2731 // Get list of firstprivate variables.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002732 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002733 auto IRef = C->varlist_begin();
2734 auto IElemInitRef = C->inits().begin();
2735 for (auto *IInit : C->private_copies()) {
2736 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2737 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataev7292c292016-04-25 12:22:29 +00002738 Data.FirstprivateVars.push_back(*IRef);
2739 Data.FirstprivateCopies.push_back(IInit);
2740 Data.FirstprivateInits.push_back(*IElemInitRef);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002741 }
Richard Trieucc3949d2016-02-18 22:34:54 +00002742 ++IRef;
2743 ++IElemInitRef;
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002744 }
2745 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00002746 // Get list of lastprivate variables (for taskloops).
2747 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2748 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2749 auto IRef = C->varlist_begin();
2750 auto ID = C->destination_exprs().begin();
2751 for (auto *IInit : C->private_copies()) {
2752 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2753 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2754 Data.LastprivateVars.push_back(*IRef);
2755 Data.LastprivateCopies.push_back(IInit);
2756 }
2757 LastprivateDstsOrigs.insert(
2758 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2759 cast<DeclRefExpr>(*IRef)});
2760 ++IRef;
2761 ++ID;
2762 }
2763 }
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00002764 SmallVector<const Expr *, 4> LHSs;
2765 SmallVector<const Expr *, 4> RHSs;
2766 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2767 auto IPriv = C->privates().begin();
2768 auto IRed = C->reduction_ops().begin();
2769 auto ILHS = C->lhs_exprs().begin();
2770 auto IRHS = C->rhs_exprs().begin();
2771 for (const auto *Ref : C->varlists()) {
2772 Data.ReductionVars.emplace_back(Ref);
2773 Data.ReductionCopies.emplace_back(*IPriv);
2774 Data.ReductionOps.emplace_back(*IRed);
2775 LHSs.emplace_back(*ILHS);
2776 RHSs.emplace_back(*IRHS);
2777 std::advance(IPriv, 1);
2778 std::advance(IRed, 1);
2779 std::advance(ILHS, 1);
2780 std::advance(IRHS, 1);
2781 }
2782 }
2783 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2784 *this, S.getLocStart(), LHSs, RHSs, Data);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00002785 // Build list of dependences.
Alexey Bataev7292c292016-04-25 12:22:29 +00002786 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2787 for (auto *IRef : C->varlists())
2788 Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00002789 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs](
Alexey Bataevf93095a2016-05-05 08:46:22 +00002790 CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002791 // Set proper addresses for generated private copies.
Alexey Bataev7292c292016-04-25 12:22:29 +00002792 OMPPrivateScope Scope(CGF);
Alexey Bataevf93095a2016-05-05 08:46:22 +00002793 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2794 !Data.LastprivateVars.empty()) {
Alexey Bataev3c595a62017-08-14 15:01:03 +00002795 enum { PrivatesParam = 2, CopyFnParam = 3 };
Alexey Bataev48591dd2016-04-20 04:01:36 +00002796 auto *CopyFn = CGF.Builder.CreateLoad(
2797 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2798 auto *PrivatesPtr = CGF.Builder.CreateLoad(
2799 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2800 // Map privates.
2801 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
2802 llvm::SmallVector<llvm::Value *, 16> CallArgs;
2803 CallArgs.push_back(PrivatesPtr);
Alexey Bataev7292c292016-04-25 12:22:29 +00002804 for (auto *E : Data.PrivateVars) {
Alexey Bataev48591dd2016-04-20 04:01:36 +00002805 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2806 Address PrivatePtr = CGF.CreateMemTemp(
2807 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2808 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2809 CallArgs.push_back(PrivatePtr.getPointer());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002810 }
Alexey Bataev7292c292016-04-25 12:22:29 +00002811 for (auto *E : Data.FirstprivateVars) {
Alexey Bataev48591dd2016-04-20 04:01:36 +00002812 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2813 Address PrivatePtr =
2814 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2815 ".firstpriv.ptr.addr");
2816 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2817 CallArgs.push_back(PrivatePtr.getPointer());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002818 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00002819 for (auto *E : Data.LastprivateVars) {
2820 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2821 Address PrivatePtr =
2822 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2823 ".lastpriv.ptr.addr");
2824 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2825 CallArgs.push_back(PrivatePtr.getPointer());
2826 }
Alexey Bataev3c595a62017-08-14 15:01:03 +00002827 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2828 CopyFn, CallArgs);
Alexey Bataevf93095a2016-05-05 08:46:22 +00002829 for (auto &&Pair : LastprivateDstsOrigs) {
2830 auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2831 DeclRefExpr DRE(
2832 const_cast<VarDecl *>(OrigVD),
2833 /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2834 OrigVD) != nullptr,
2835 Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2836 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2837 return CGF.EmitLValue(&DRE).getAddress();
2838 });
2839 }
Alexey Bataev48591dd2016-04-20 04:01:36 +00002840 for (auto &&Pair : PrivatePtrs) {
2841 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2842 CGF.getContext().getDeclAlign(Pair.first));
2843 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2844 }
Alexey Bataev3ae88e22015-05-22 08:56:35 +00002845 }
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00002846 if (Data.Reductions) {
2847 OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true);
2848 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
2849 Data.ReductionOps);
2850 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2851 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2852 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2853 RedCG.emitSharedLValue(CGF, Cnt);
2854 RedCG.emitAggregateType(CGF, Cnt);
2855 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2856 CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2857 Replacement =
2858 Address(CGF.EmitScalarConversion(
2859 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2860 CGF.getContext().getPointerType(
2861 Data.ReductionCopies[Cnt]->getType()),
2862 SourceLocation()),
2863 Replacement.getAlignment());
2864 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2865 Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2866 [Replacement]() { return Replacement; });
2867 // FIXME: This must removed once the runtime library is fixed.
2868 // Emit required threadprivate variables for
2869 // initilizer/combiner/finalizer.
2870 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2871 RedCG, Cnt);
2872 }
2873 }
Alexey Bataev88202be2017-07-27 13:20:36 +00002874 // Privatize all private variables except for in_reduction items.
Alexey Bataev48591dd2016-04-20 04:01:36 +00002875 (void)Scope.Privatize();
Alexey Bataev88202be2017-07-27 13:20:36 +00002876 SmallVector<const Expr *, 4> InRedVars;
2877 SmallVector<const Expr *, 4> InRedPrivs;
2878 SmallVector<const Expr *, 4> InRedOps;
2879 SmallVector<const Expr *, 4> TaskgroupDescriptors;
2880 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2881 auto IPriv = C->privates().begin();
2882 auto IRed = C->reduction_ops().begin();
2883 auto ITD = C->taskgroup_descriptors().begin();
2884 for (const auto *Ref : C->varlists()) {
2885 InRedVars.emplace_back(Ref);
2886 InRedPrivs.emplace_back(*IPriv);
2887 InRedOps.emplace_back(*IRed);
2888 TaskgroupDescriptors.emplace_back(*ITD);
2889 std::advance(IPriv, 1);
2890 std::advance(IRed, 1);
2891 std::advance(ITD, 1);
2892 }
2893 }
2894 // Privatize in_reduction items here, because taskgroup descriptors must be
2895 // privatized earlier.
2896 OMPPrivateScope InRedScope(CGF);
2897 if (!InRedVars.empty()) {
2898 ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
2899 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2900 RedCG.emitSharedLValue(CGF, Cnt);
2901 RedCG.emitAggregateType(CGF, Cnt);
2902 // The taskgroup descriptor variable is always implicit firstprivate and
2903 // privatized already during procoessing of the firstprivates.
2904 llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(
2905 CGF.EmitLValue(TaskgroupDescriptors[Cnt]), SourceLocation());
2906 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2907 CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2908 Replacement = Address(
2909 CGF.EmitScalarConversion(
2910 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2911 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2912 SourceLocation()),
2913 Replacement.getAlignment());
2914 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2915 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
2916 [Replacement]() { return Replacement; });
2917 // FIXME: This must removed once the runtime library is fixed.
2918 // Emit required threadprivate variables for
2919 // initilizer/combiner/finalizer.
2920 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2921 RedCG, Cnt);
2922 }
2923 }
2924 (void)InRedScope.Privatize();
Alexey Bataev48591dd2016-04-20 04:01:36 +00002925
2926 Action.Enter(CGF);
Alexey Bataev7292c292016-04-25 12:22:29 +00002927 BodyGen(CGF);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002928 };
Alexey Bataev7292c292016-04-25 12:22:29 +00002929 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2930 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2931 Data.NumberOfParts);
2932 OMPLexicalScope Scope(*this, S);
2933 TaskGen(*this, OutlinedFn, Data);
2934}
2935
2936void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
2937 // Emit outlined function for task construct.
2938 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2939 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
Alexey Bataev62b63b12015-03-10 07:28:44 +00002940 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
Alexey Bataev1d677132015-04-22 13:57:31 +00002941 const Expr *IfCond = nullptr;
Alexey Bataev7371aa32015-09-03 08:45:56 +00002942 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2943 if (C->getNameModifier() == OMPD_unknown ||
2944 C->getNameModifier() == OMPD_task) {
2945 IfCond = C->getCondition();
2946 break;
2947 }
Alexey Bataev1d677132015-04-22 13:57:31 +00002948 }
Alexey Bataev7292c292016-04-25 12:22:29 +00002949
Alexey Bataev24b5bae2016-04-28 09:23:51 +00002950 OMPTaskDataTy Data;
2951 // Check if we should emit tied or untied task.
2952 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
Alexey Bataev7292c292016-04-25 12:22:29 +00002953 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
2954 CGF.EmitStmt(CS->getCapturedStmt());
2955 };
Alexey Bataev24b5bae2016-04-28 09:23:51 +00002956 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
Alexey Bataev7292c292016-04-25 12:22:29 +00002957 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
Alexey Bataev24b5bae2016-04-28 09:23:51 +00002958 const OMPTaskDataTy &Data) {
2959 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
2960 SharedsTy, CapturedStruct, IfCond,
2961 Data);
Alexey Bataev7292c292016-04-25 12:22:29 +00002962 };
Alexey Bataev24b5bae2016-04-28 09:23:51 +00002963 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
Alexey Bataev9c2e8ee2014-07-11 11:25:16 +00002964}
2965
Alexey Bataev9f797f32015-02-05 05:57:51 +00002966void CodeGenFunction::EmitOMPTaskyieldDirective(
2967 const OMPTaskyieldDirective &S) {
Alexey Bataev3eff5f42015-02-25 08:32:46 +00002968 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
Alexey Bataev68446b72014-07-18 07:47:19 +00002969}
2970
Alexey Bataev8f7c1b02014-12-05 04:09:23 +00002971void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
Alexey Bataevf2685682015-03-30 04:30:22 +00002972 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
Alexey Bataev4d1dfea2014-07-18 09:11:51 +00002973}
2974
Alexey Bataev8b8e2022015-04-27 05:22:09 +00002975void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
2976 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
Alexey Bataev2df347a2014-07-18 10:17:07 +00002977}
2978
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00002979void CodeGenFunction::EmitOMPTaskgroupDirective(
2980 const OMPTaskgroupDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002981 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2982 Action.Enter(CGF);
Alexey Bataev3b1b8952017-07-25 15:53:26 +00002983 if (const Expr *E = S.getReductionRef()) {
2984 SmallVector<const Expr *, 4> LHSs;
2985 SmallVector<const Expr *, 4> RHSs;
2986 OMPTaskDataTy Data;
2987 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
2988 auto IPriv = C->privates().begin();
2989 auto IRed = C->reduction_ops().begin();
2990 auto ILHS = C->lhs_exprs().begin();
2991 auto IRHS = C->rhs_exprs().begin();
2992 for (const auto *Ref : C->varlists()) {
2993 Data.ReductionVars.emplace_back(Ref);
2994 Data.ReductionCopies.emplace_back(*IPriv);
2995 Data.ReductionOps.emplace_back(*IRed);
2996 LHSs.emplace_back(*ILHS);
2997 RHSs.emplace_back(*IRHS);
2998 std::advance(IPriv, 1);
2999 std::advance(IRed, 1);
3000 std::advance(ILHS, 1);
3001 std::advance(IRHS, 1);
3002 }
3003 }
3004 llvm::Value *ReductionDesc =
3005 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(),
3006 LHSs, RHSs, Data);
3007 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3008 CGF.EmitVarDecl(*VD);
3009 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3010 /*Volatile=*/false, E->getType());
3011 }
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00003012 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00003013 };
Alexey Bataev4ba78a42016-04-27 07:56:03 +00003014 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00003015 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
3016}
3017
Alexey Bataevcc37cc12014-11-20 04:34:54 +00003018void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003019 CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00003020 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) {
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003021 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3022 FlushClause->varlist_end());
3023 }
3024 return llvm::None;
3025 }(), S.getLocStart());
Alexey Bataev6125da92014-07-21 11:26:11 +00003026}
3027
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003028void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
3029 const CodeGenLoopTy &CodeGenLoop,
3030 Expr *IncExpr) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003031 // Emit the loop iteration variable.
3032 auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3033 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
3034 EmitVarDecl(*IVDecl);
3035
3036 // Emit the iterations count variable.
3037 // If it is not a variable, Sema decided to calculate iterations count on each
3038 // iteration (e.g., it is foldable into a constant).
3039 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3040 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3041 // Emit calculation of the iterations count.
3042 EmitIgnoredExpr(S.getCalcLastIteration());
3043 }
3044
3045 auto &RT = CGM.getOpenMPRuntime();
3046
Carlo Bertolli962bb802017-01-03 18:24:42 +00003047 bool HasLastprivateClause = false;
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003048 // Check pre-condition.
3049 {
Alexey Bataev5a3af132016-03-29 08:58:54 +00003050 OMPLoopScope PreInitScope(*this, S);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003051 // Skip the entire loop if we don't meet the precondition.
3052 // If the condition constant folds and can be elided, avoid emitting the
3053 // whole loop.
3054 bool CondConstant;
3055 llvm::BasicBlock *ContBlock = nullptr;
3056 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3057 if (!CondConstant)
3058 return;
3059 } else {
3060 auto *ThenBlock = createBasicBlock("omp.precond.then");
3061 ContBlock = createBasicBlock("omp.precond.end");
3062 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3063 getProfileCount(&S));
3064 EmitBlock(ThenBlock);
3065 incrementProfileCounter(&S);
3066 }
3067
Alexey Bataev617db5f2017-12-04 15:38:33 +00003068 emitAlignedClause(*this, S);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003069 // Emit 'then' code.
3070 {
3071 // Emit helper vars inits.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003072
3073 LValue LB = EmitOMPHelperVar(
3074 *this, cast<DeclRefExpr>(
3075 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3076 ? S.getCombinedLowerBoundVariable()
3077 : S.getLowerBoundVariable())));
3078 LValue UB = EmitOMPHelperVar(
3079 *this, cast<DeclRefExpr>(
3080 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3081 ? S.getCombinedUpperBoundVariable()
3082 : S.getUpperBoundVariable())));
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003083 LValue ST =
3084 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3085 LValue IL =
3086 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3087
3088 OMPPrivateScope LoopScope(*this);
Carlo Bertolli962bb802017-01-03 18:24:42 +00003089 if (EmitOMPFirstprivateClause(S, LoopScope)) {
Alexey Bataev617db5f2017-12-04 15:38:33 +00003090 // Emit implicit barrier to synchronize threads and avoid data races
3091 // on initialization of firstprivate variables and post-update of
Carlo Bertolli962bb802017-01-03 18:24:42 +00003092 // lastprivate variables.
3093 CGM.getOpenMPRuntime().emitBarrierCall(
Alexey Bataev617db5f2017-12-04 15:38:33 +00003094 *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
3095 /*ForceSimpleCall=*/true);
Carlo Bertolli962bb802017-01-03 18:24:42 +00003096 }
3097 EmitOMPPrivateClause(S, LoopScope);
Alexey Bataev617db5f2017-12-04 15:38:33 +00003098 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3099 !isOpenMPParallelDirective(S.getDirectiveKind()))
3100 EmitOMPReductionClauseInit(S, LoopScope);
Carlo Bertolli962bb802017-01-03 18:24:42 +00003101 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00003102 EmitOMPPrivateLoopCounters(S, LoopScope);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003103 (void)LoopScope.Privatize();
3104
3105 // Detect the distribute schedule kind and chunk.
3106 llvm::Value *Chunk = nullptr;
3107 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
3108 if (auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3109 ScheduleKind = C->getDistScheduleKind();
3110 if (const auto *Ch = C->getChunkSize()) {
3111 Chunk = EmitScalarExpr(Ch);
3112 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
Alexey Bataev617db5f2017-12-04 15:38:33 +00003113 S.getIterationVariable()->getType(),
3114 S.getLocStart());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003115 }
3116 }
3117 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3118 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3119
3120 // OpenMP [2.10.8, distribute Construct, Description]
3121 // If dist_schedule is specified, kind must be static. If specified,
3122 // iterations are divided into chunks of size chunk_size, chunks are
3123 // assigned to the teams of the league in a round-robin fashion in the
3124 // order of the team number. When no chunk_size is specified, the
3125 // iteration space is divided into chunks that are approximately equal
3126 // in size, and at most one chunk is distributed to each team of the
3127 // league. The size of the chunks is unspecified in this case.
3128 if (RT.isStaticNonchunked(ScheduleKind,
3129 /* Chunked */ Chunk != nullptr)) {
Alexey Bataev617db5f2017-12-04 15:38:33 +00003130 if (isOpenMPSimdDirective(S.getDirectiveKind()))
3131 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003132 CGOpenMPRuntime::StaticRTInput StaticInit(
3133 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3134 LB.getAddress(), UB.getAddress(), ST.getAddress());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003135 RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003136 StaticInit);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003137 auto LoopExit =
3138 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3139 // UB = min(UB, GlobalUB);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003140 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3141 ? S.getCombinedEnsureUpperBound()
3142 : S.getEnsureUpperBound());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003143 // IV = LB;
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003144 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3145 ? S.getCombinedInit()
3146 : S.getInit());
3147
3148 Expr *Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3149 ? S.getCombinedCond()
3150 : S.getCond();
3151
3152 // for distribute alone, codegen
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003153 // while (idx <= UB) { BODY; ++idx; }
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003154 // when combined with 'for' (e.g. as in 'distribute parallel for')
3155 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3156 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3157 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3158 CodeGenLoop(CGF, S, LoopExit);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003159 },
3160 [](CodeGenFunction &) {});
3161 EmitBlock(LoopExit.getBlock());
3162 // Tell the runtime we are done.
Alexey Bataevf43f7142017-09-06 16:17:35 +00003163 RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003164 } else {
3165 // Emit the outer loop, which requests its work chunk [LB..UB] from
3166 // runtime and runs the inner loop to process it.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003167 const OMPLoopArguments LoopArguments = {
3168 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3169 Chunk};
3170 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3171 CodeGenLoop);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003172 }
Alexey Bataev617db5f2017-12-04 15:38:33 +00003173 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3174 EmitOMPSimdFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3175 return CGF.Builder.CreateIsNotNull(
3176 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3177 });
3178 }
3179 OpenMPDirectiveKind ReductionKind = OMPD_unknown;
3180 if (isOpenMPParallelDirective(S.getDirectiveKind()) &&
3181 isOpenMPSimdDirective(S.getDirectiveKind())) {
3182 ReductionKind = OMPD_parallel_for_simd;
3183 } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3184 ReductionKind = OMPD_parallel_for;
3185 } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3186 ReductionKind = OMPD_simd;
3187 } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
3188 S.hasClausesOfKind<OMPReductionClause>()) {
3189 llvm_unreachable(
3190 "No reduction clauses is allowed in distribute directive.");
3191 }
3192 EmitOMPReductionClauseFinal(S, ReductionKind);
3193 // Emit post-update of the reduction variables if IsLastIter != 0.
3194 emitPostUpdateForReductionClause(
3195 *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3196 return CGF.Builder.CreateIsNotNull(
3197 CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3198 });
Carlo Bertolli962bb802017-01-03 18:24:42 +00003199 // Emit final copy of the lastprivate variables if IsLastIter != 0.
Alexey Bataev617db5f2017-12-04 15:38:33 +00003200 if (HasLastprivateClause) {
Carlo Bertolli962bb802017-01-03 18:24:42 +00003201 EmitOMPLastprivateClauseFinal(
3202 S, /*NoFinals=*/false,
Alexey Bataev617db5f2017-12-04 15:38:33 +00003203 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
3204 }
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003205 }
3206
3207 // We're now done with the loop, so jump to the continuation block.
3208 if (ContBlock) {
3209 EmitBranch(ContBlock);
3210 EmitBlock(ContBlock, true);
3211 }
3212 }
3213}
3214
Carlo Bertolli6200a3d2015-12-14 14:51:25 +00003215void CodeGenFunction::EmitOMPDistributeDirective(
3216 const OMPDistributeDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003217 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003218
3219 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003220 };
Alexey Bataev4ba78a42016-04-27 07:56:03 +00003221 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev10a54312017-11-27 16:54:08 +00003222 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
Carlo Bertolli6200a3d2015-12-14 14:51:25 +00003223}
3224
Alexey Bataev5f600d62015-09-29 03:48:57 +00003225static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3226 const CapturedStmt *S) {
3227 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3228 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3229 CGF.CapturedStmtInfo = &CapStmtInfo;
3230 auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3231 Fn->addFnAttr(llvm::Attribute::NoInline);
3232 return Fn;
3233}
3234
Alexey Bataev98eb6e32015-04-22 11:15:40 +00003235void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
Alexey Bataev8b427062016-05-25 12:36:08 +00003236 if (!S.getAssociatedStmt()) {
3237 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3238 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
Alexey Bataev8ef31412015-12-18 07:58:25 +00003239 return;
Alexey Bataev8b427062016-05-25 12:36:08 +00003240 }
Alexey Bataev5f600d62015-09-29 03:48:57 +00003241 auto *C = S.getSingleClause<OMPSIMDClause>();
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003242 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3243 PrePostActionTy &Action) {
Alexey Bataev5f600d62015-09-29 03:48:57 +00003244 if (C) {
3245 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3246 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3247 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3248 auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
Alexey Bataev3c595a62017-08-14 15:01:03 +00003249 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3250 OutlinedFn, CapturedVars);
Alexey Bataev5f600d62015-09-29 03:48:57 +00003251 } else {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003252 Action.Enter(CGF);
Alexey Bataev5f600d62015-09-29 03:48:57 +00003253 CGF.EmitStmt(
3254 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3255 }
Alexey Bataev98eb6e32015-04-22 11:15:40 +00003256 };
Alexey Bataev4ba78a42016-04-27 07:56:03 +00003257 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev5f600d62015-09-29 03:48:57 +00003258 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
Alexey Bataev9fb6e642014-07-22 06:45:04 +00003259}
3260
Alexey Bataevb57056f2015-01-22 06:17:56 +00003261static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003262 QualType SrcType, QualType DestType,
3263 SourceLocation Loc) {
Alexey Bataevb57056f2015-01-22 06:17:56 +00003264 assert(CGF.hasScalarEvaluationKind(DestType) &&
3265 "DestType must have scalar evaluation kind.");
3266 assert(!Val.isAggregate() && "Must be a scalar or complex.");
3267 return Val.isScalar()
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003268 ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
3269 Loc)
Alexey Bataevb57056f2015-01-22 06:17:56 +00003270 : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003271 DestType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003272}
3273
3274static CodeGenFunction::ComplexPairTy
3275convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003276 QualType DestType, SourceLocation Loc) {
Alexey Bataevb57056f2015-01-22 06:17:56 +00003277 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3278 "DestType must have complex evaluation kind.");
3279 CodeGenFunction::ComplexPairTy ComplexVal;
3280 if (Val.isScalar()) {
3281 // Convert the input element to the element type of the complex.
3282 auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003283 auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3284 DestElementType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003285 ComplexVal = CodeGenFunction::ComplexPairTy(
3286 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3287 } else {
3288 assert(Val.isComplex() && "Must be a scalar or complex.");
3289 auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3290 auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3291 ComplexVal.first = CGF.EmitScalarConversion(
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003292 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003293 ComplexVal.second = CGF.EmitScalarConversion(
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003294 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003295 }
3296 return ComplexVal;
3297}
3298
Alexey Bataev5e018f92015-04-23 06:35:10 +00003299static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3300 LValue LVal, RValue RVal) {
3301 if (LVal.isGlobalReg()) {
3302 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3303 } else {
JF Bastien92f4ef12016-04-06 17:26:42 +00003304 CGF.EmitAtomicStore(RVal, LVal,
3305 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3306 : llvm::AtomicOrdering::Monotonic,
Alexey Bataev5e018f92015-04-23 06:35:10 +00003307 LVal.isVolatile(), /*IsInit=*/false);
3308 }
3309}
3310
Alexey Bataev8524d152016-01-21 12:35:58 +00003311void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3312 QualType RValTy, SourceLocation Loc) {
3313 switch (getEvaluationKind(LVal.getType())) {
Alexey Bataev5e018f92015-04-23 06:35:10 +00003314 case TEK_Scalar:
Alexey Bataev8524d152016-01-21 12:35:58 +00003315 EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3316 *this, RVal, RValTy, LVal.getType(), Loc)),
3317 LVal);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003318 break;
3319 case TEK_Complex:
Alexey Bataev8524d152016-01-21 12:35:58 +00003320 EmitStoreOfComplex(
3321 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
Alexey Bataev5e018f92015-04-23 06:35:10 +00003322 /*isInit=*/false);
3323 break;
3324 case TEK_Aggregate:
3325 llvm_unreachable("Must be a scalar or complex.");
3326 }
3327}
3328
Alexey Bataevb57056f2015-01-22 06:17:56 +00003329static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3330 const Expr *X, const Expr *V,
3331 SourceLocation Loc) {
3332 // v = x;
3333 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3334 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3335 LValue XLValue = CGF.EmitLValue(X);
3336 LValue VLValue = CGF.EmitLValue(V);
David Majnemera5b195a2015-02-14 01:35:12 +00003337 RValue Res = XLValue.isGlobalReg()
3338 ? CGF.EmitLoadOfLValue(XLValue, Loc)
JF Bastien92f4ef12016-04-06 17:26:42 +00003339 : CGF.EmitAtomicLoad(
3340 XLValue, Loc,
3341 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3342 : llvm::AtomicOrdering::Monotonic,
3343 XLValue.isVolatile());
Alexey Bataevb57056f2015-01-22 06:17:56 +00003344 // OpenMP, 2.12.6, atomic Construct
3345 // Any atomic construct with a seq_cst clause forces the atomically
3346 // performed operation to include an implicit flush operation without a
3347 // list.
3348 if (IsSeqCst)
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003349 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
Alexey Bataev8524d152016-01-21 12:35:58 +00003350 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003351}
3352
Alexey Bataevb8329262015-02-27 06:33:30 +00003353static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3354 const Expr *X, const Expr *E,
3355 SourceLocation Loc) {
3356 // x = expr;
3357 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
Alexey Bataev5e018f92015-04-23 06:35:10 +00003358 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
Alexey Bataevb8329262015-02-27 06:33:30 +00003359 // OpenMP, 2.12.6, atomic Construct
3360 // Any atomic construct with a seq_cst clause forces the atomically
3361 // performed operation to include an implicit flush operation without a
3362 // list.
3363 if (IsSeqCst)
3364 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3365}
3366
Benjamin Kramer439ee9d2015-05-01 13:59:53 +00003367static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3368 RValue Update,
3369 BinaryOperatorKind BO,
3370 llvm::AtomicOrdering AO,
3371 bool IsXLHSInRHSPart) {
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003372 auto &Context = CGF.CGM.getContext();
3373 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
Alexey Bataevb4505a72015-03-30 05:20:59 +00003374 // expression is simple and atomic is allowed for the given type for the
3375 // target platform.
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003376 if (BO == BO_Comma || !Update.isScalar() ||
Alexey Bataev9d541a72015-05-08 11:47:16 +00003377 !Update.getScalarVal()->getType()->isIntegerTy() ||
3378 !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3379 (Update.getScalarVal()->getType() !=
John McCall7f416cc2015-09-08 08:05:57 +00003380 X.getAddress().getElementType())) ||
3381 !X.getAddress().getElementType()->isIntegerTy() ||
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003382 !Context.getTargetInfo().hasBuiltinAtomic(
3383 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
Alexey Bataev5e018f92015-04-23 06:35:10 +00003384 return std::make_pair(false, RValue::get(nullptr));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003385
3386 llvm::AtomicRMWInst::BinOp RMWOp;
3387 switch (BO) {
3388 case BO_Add:
3389 RMWOp = llvm::AtomicRMWInst::Add;
3390 break;
3391 case BO_Sub:
3392 if (!IsXLHSInRHSPart)
Alexey Bataev5e018f92015-04-23 06:35:10 +00003393 return std::make_pair(false, RValue::get(nullptr));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003394 RMWOp = llvm::AtomicRMWInst::Sub;
3395 break;
3396 case BO_And:
3397 RMWOp = llvm::AtomicRMWInst::And;
3398 break;
3399 case BO_Or:
3400 RMWOp = llvm::AtomicRMWInst::Or;
3401 break;
3402 case BO_Xor:
3403 RMWOp = llvm::AtomicRMWInst::Xor;
3404 break;
3405 case BO_LT:
3406 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3407 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3408 : llvm::AtomicRMWInst::Max)
3409 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3410 : llvm::AtomicRMWInst::UMax);
3411 break;
3412 case BO_GT:
3413 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3414 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3415 : llvm::AtomicRMWInst::Min)
3416 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3417 : llvm::AtomicRMWInst::UMin);
3418 break;
Alexey Bataev5e018f92015-04-23 06:35:10 +00003419 case BO_Assign:
3420 RMWOp = llvm::AtomicRMWInst::Xchg;
3421 break;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003422 case BO_Mul:
3423 case BO_Div:
3424 case BO_Rem:
3425 case BO_Shl:
3426 case BO_Shr:
3427 case BO_LAnd:
3428 case BO_LOr:
Alexey Bataev5e018f92015-04-23 06:35:10 +00003429 return std::make_pair(false, RValue::get(nullptr));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003430 case BO_PtrMemD:
3431 case BO_PtrMemI:
3432 case BO_LE:
3433 case BO_GE:
3434 case BO_EQ:
3435 case BO_NE:
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003436 case BO_AddAssign:
3437 case BO_SubAssign:
3438 case BO_AndAssign:
3439 case BO_OrAssign:
3440 case BO_XorAssign:
3441 case BO_MulAssign:
3442 case BO_DivAssign:
3443 case BO_RemAssign:
3444 case BO_ShlAssign:
3445 case BO_ShrAssign:
3446 case BO_Comma:
3447 llvm_unreachable("Unsupported atomic update operation");
3448 }
3449 auto *UpdateVal = Update.getScalarVal();
3450 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3451 UpdateVal = CGF.Builder.CreateIntCast(
John McCall7f416cc2015-09-08 08:05:57 +00003452 IC, X.getAddress().getElementType(),
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003453 X.getType()->hasSignedIntegerRepresentation());
3454 }
John McCall7f416cc2015-09-08 08:05:57 +00003455 auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003456 return std::make_pair(true, RValue::get(Res));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003457}
3458
Alexey Bataev5e018f92015-04-23 06:35:10 +00003459std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003460 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3461 llvm::AtomicOrdering AO, SourceLocation Loc,
3462 const llvm::function_ref<RValue(RValue)> &CommonGen) {
3463 // Update expressions are allowed to have the following forms:
3464 // x binop= expr; -> xrval + expr;
3465 // x++, ++x -> xrval + 1;
3466 // x--, --x -> xrval - 1;
3467 // x = x binop expr; -> xrval binop expr
3468 // x = expr Op x; - > expr binop xrval;
Alexey Bataev5e018f92015-04-23 06:35:10 +00003469 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3470 if (!Res.first) {
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003471 if (X.isGlobalReg()) {
3472 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3473 // 'xrval'.
3474 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3475 } else {
3476 // Perform compare-and-swap procedure.
3477 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
Alexey Bataevb4505a72015-03-30 05:20:59 +00003478 }
3479 }
Alexey Bataev5e018f92015-04-23 06:35:10 +00003480 return Res;
Alexey Bataevb4505a72015-03-30 05:20:59 +00003481}
3482
3483static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3484 const Expr *X, const Expr *E,
3485 const Expr *UE, bool IsXLHSInRHSPart,
3486 SourceLocation Loc) {
3487 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3488 "Update expr in 'atomic update' must be a binary operator.");
3489 auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3490 // Update expressions are allowed to have the following forms:
3491 // x binop= expr; -> xrval + expr;
3492 // x++, ++x -> xrval + 1;
3493 // x--, --x -> xrval - 1;
3494 // x = x binop expr; -> xrval binop expr
3495 // x = expr Op x; - > expr binop xrval;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003496 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
Alexey Bataevb4505a72015-03-30 05:20:59 +00003497 LValue XLValue = CGF.EmitLValue(X);
3498 RValue ExprRValue = CGF.EmitAnyExpr(E);
JF Bastien92f4ef12016-04-06 17:26:42 +00003499 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3500 : llvm::AtomicOrdering::Monotonic;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003501 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3502 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3503 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3504 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3505 auto Gen =
3506 [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue {
3507 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3508 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3509 return CGF.EmitAnyExpr(UE);
3510 };
Alexey Bataev5e018f92015-04-23 06:35:10 +00003511 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
3512 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3513 // OpenMP, 2.12.6, atomic Construct
3514 // Any atomic construct with a seq_cst clause forces the atomically
3515 // performed operation to include an implicit flush operation without a
3516 // list.
3517 if (IsSeqCst)
3518 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3519}
3520
3521static RValue convertToType(CodeGenFunction &CGF, RValue Value,
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003522 QualType SourceType, QualType ResType,
3523 SourceLocation Loc) {
Alexey Bataev5e018f92015-04-23 06:35:10 +00003524 switch (CGF.getEvaluationKind(ResType)) {
3525 case TEK_Scalar:
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003526 return RValue::get(
3527 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
Alexey Bataev5e018f92015-04-23 06:35:10 +00003528 case TEK_Complex: {
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003529 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003530 return RValue::getComplex(Res.first, Res.second);
3531 }
3532 case TEK_Aggregate:
3533 break;
3534 }
3535 llvm_unreachable("Must be a scalar or complex.");
3536}
3537
3538static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3539 bool IsPostfixUpdate, const Expr *V,
3540 const Expr *X, const Expr *E,
3541 const Expr *UE, bool IsXLHSInRHSPart,
3542 SourceLocation Loc) {
3543 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3544 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3545 RValue NewVVal;
3546 LValue VLValue = CGF.EmitLValue(V);
3547 LValue XLValue = CGF.EmitLValue(X);
3548 RValue ExprRValue = CGF.EmitAnyExpr(E);
JF Bastien92f4ef12016-04-06 17:26:42 +00003549 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3550 : llvm::AtomicOrdering::Monotonic;
Alexey Bataev5e018f92015-04-23 06:35:10 +00003551 QualType NewVValType;
3552 if (UE) {
3553 // 'x' is updated with some additional value.
3554 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3555 "Update expr in 'atomic capture' must be a binary operator.");
3556 auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3557 // Update expressions are allowed to have the following forms:
3558 // x binop= expr; -> xrval + expr;
3559 // x++, ++x -> xrval + 1;
3560 // x--, --x -> xrval - 1;
3561 // x = x binop expr; -> xrval binop expr
3562 // x = expr Op x; - > expr binop xrval;
3563 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3564 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3565 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3566 NewVValType = XRValExpr->getType();
3567 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3568 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00003569 IsPostfixUpdate](RValue XRValue) -> RValue {
Alexey Bataev5e018f92015-04-23 06:35:10 +00003570 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3571 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3572 RValue Res = CGF.EmitAnyExpr(UE);
3573 NewVVal = IsPostfixUpdate ? XRValue : Res;
3574 return Res;
3575 };
3576 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3577 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3578 if (Res.first) {
3579 // 'atomicrmw' instruction was generated.
3580 if (IsPostfixUpdate) {
3581 // Use old value from 'atomicrmw'.
3582 NewVVal = Res.second;
3583 } else {
3584 // 'atomicrmw' does not provide new value, so evaluate it using old
3585 // value of 'x'.
3586 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3587 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3588 NewVVal = CGF.EmitAnyExpr(UE);
3589 }
3590 }
3591 } else {
3592 // 'x' is simply rewritten with some 'expr'.
3593 NewVValType = X->getType().getNonReferenceType();
3594 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003595 X->getType().getNonReferenceType(), Loc);
Malcolm Parsonsc6e45832017-01-13 18:55:32 +00003596 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) -> RValue {
Alexey Bataev5e018f92015-04-23 06:35:10 +00003597 NewVVal = XRValue;
3598 return ExprRValue;
3599 };
3600 // Try to perform atomicrmw xchg, otherwise simple exchange.
3601 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3602 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3603 Loc, Gen);
3604 if (Res.first) {
3605 // 'atomicrmw' instruction was generated.
3606 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3607 }
3608 }
3609 // Emit post-update store to 'v' of old/new 'x' value.
Alexey Bataev8524d152016-01-21 12:35:58 +00003610 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
Alexey Bataevb4505a72015-03-30 05:20:59 +00003611 // OpenMP, 2.12.6, atomic Construct
3612 // Any atomic construct with a seq_cst clause forces the atomically
3613 // performed operation to include an implicit flush operation without a
3614 // list.
3615 if (IsSeqCst)
3616 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3617}
3618
Alexey Bataevb57056f2015-01-22 06:17:56 +00003619static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
Alexey Bataev5e018f92015-04-23 06:35:10 +00003620 bool IsSeqCst, bool IsPostfixUpdate,
3621 const Expr *X, const Expr *V, const Expr *E,
3622 const Expr *UE, bool IsXLHSInRHSPart,
3623 SourceLocation Loc) {
Alexey Bataevb57056f2015-01-22 06:17:56 +00003624 switch (Kind) {
3625 case OMPC_read:
3626 EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3627 break;
3628 case OMPC_write:
Alexey Bataevb8329262015-02-27 06:33:30 +00003629 EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3630 break;
Alexey Bataevb4505a72015-03-30 05:20:59 +00003631 case OMPC_unknown:
Alexey Bataevb57056f2015-01-22 06:17:56 +00003632 case OMPC_update:
Alexey Bataevb4505a72015-03-30 05:20:59 +00003633 EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3634 break;
Alexey Bataevb57056f2015-01-22 06:17:56 +00003635 case OMPC_capture:
Alexey Bataev5e018f92015-04-23 06:35:10 +00003636 EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3637 IsXLHSInRHSPart, Loc);
3638 break;
Alexey Bataevb57056f2015-01-22 06:17:56 +00003639 case OMPC_if:
3640 case OMPC_final:
3641 case OMPC_num_threads:
3642 case OMPC_private:
3643 case OMPC_firstprivate:
3644 case OMPC_lastprivate:
3645 case OMPC_reduction:
Alexey Bataev169d96a2017-07-18 20:17:46 +00003646 case OMPC_task_reduction:
Alexey Bataevfa312f32017-07-21 18:48:21 +00003647 case OMPC_in_reduction:
Alexey Bataevb57056f2015-01-22 06:17:56 +00003648 case OMPC_safelen:
Alexey Bataev66b15b52015-08-21 11:14:16 +00003649 case OMPC_simdlen:
Alexey Bataevb57056f2015-01-22 06:17:56 +00003650 case OMPC_collapse:
3651 case OMPC_default:
3652 case OMPC_seq_cst:
3653 case OMPC_shared:
3654 case OMPC_linear:
3655 case OMPC_aligned:
3656 case OMPC_copyin:
3657 case OMPC_copyprivate:
3658 case OMPC_flush:
3659 case OMPC_proc_bind:
3660 case OMPC_schedule:
3661 case OMPC_ordered:
3662 case OMPC_nowait:
3663 case OMPC_untied:
3664 case OMPC_threadprivate:
Alexey Bataev1c2cfbc2015-06-23 14:25:19 +00003665 case OMPC_depend:
Alexey Bataevb57056f2015-01-22 06:17:56 +00003666 case OMPC_mergeable:
Michael Wonge710d542015-08-07 16:16:36 +00003667 case OMPC_device:
Alexey Bataev346265e2015-09-25 10:37:12 +00003668 case OMPC_threads:
Alexey Bataevd14d1e62015-09-28 06:39:35 +00003669 case OMPC_simd:
Kelvin Li0bff7af2015-11-23 05:32:03 +00003670 case OMPC_map:
Kelvin Li099bb8c2015-11-24 20:50:12 +00003671 case OMPC_num_teams:
Kelvin Lia15fb1a2015-11-27 18:47:36 +00003672 case OMPC_thread_limit:
Alexey Bataeva0569352015-12-01 10:17:31 +00003673 case OMPC_priority:
Alexey Bataev1fd4aed2015-12-07 12:52:51 +00003674 case OMPC_grainsize:
Alexey Bataevb825de12015-12-07 10:51:44 +00003675 case OMPC_nogroup:
Alexey Bataev382967a2015-12-08 12:06:20 +00003676 case OMPC_num_tasks:
Alexey Bataev28c75412015-12-15 08:19:24 +00003677 case OMPC_hint:
Carlo Bertollib4adf552016-01-15 18:50:31 +00003678 case OMPC_dist_schedule:
Arpith Chacko Jacob3cf89042016-01-26 16:37:23 +00003679 case OMPC_defaultmap:
Alexey Bataeve48a5fc2016-04-12 05:28:34 +00003680 case OMPC_uniform:
Samuel Antao661c0902016-05-26 17:39:58 +00003681 case OMPC_to:
Samuel Antaoec172c62016-05-26 17:49:04 +00003682 case OMPC_from:
Carlo Bertolli2404b172016-07-13 15:37:16 +00003683 case OMPC_use_device_ptr:
Carlo Bertolli70594e92016-07-13 17:16:49 +00003684 case OMPC_is_device_ptr:
Alexey Bataevb57056f2015-01-22 06:17:56 +00003685 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3686 }
3687}
3688
3689void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00003690 bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
Alexey Bataevb57056f2015-01-22 06:17:56 +00003691 OpenMPClauseKind Kind = OMPC_unknown;
3692 for (auto *C : S.clauses()) {
3693 // Find first clause (skip seq_cst clause, if it is first).
3694 if (C->getClauseKind() != OMPC_seq_cst) {
3695 Kind = C->getClauseKind();
3696 break;
3697 }
3698 }
Alexey Bataev10fec572015-03-11 04:48:56 +00003699
3700 const auto *CS =
3701 S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003702 if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
Alexey Bataev10fec572015-03-11 04:48:56 +00003703 enterFullExpression(EWC);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003704 }
3705 // Processing for statements under 'atomic capture'.
3706 if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3707 for (const auto *C : Compound->body()) {
3708 if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
3709 enterFullExpression(EWC);
3710 }
3711 }
3712 }
Alexey Bataev10fec572015-03-11 04:48:56 +00003713
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003714 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3715 PrePostActionTy &) {
Alexey Bataev33c56402015-12-14 09:26:19 +00003716 CGF.EmitStopPoint(CS);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003717 EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3718 S.getV(), S.getExpr(), S.getUpdateExpr(),
3719 S.isXLHSInRHSPart(), S.getLocStart());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00003720 };
Alexey Bataev4ba78a42016-04-27 07:56:03 +00003721 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00003722 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
Alexey Bataev0162e452014-07-22 10:10:35 +00003723}
3724
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003725static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
3726 const OMPExecutableDirective &S,
3727 const RegionCodeGenTy &CodeGen) {
3728 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
3729 CodeGenModule &CGM = CGF.CGM;
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00003730 const CapturedStmt &CS = *S.getCapturedStmt(OMPD_target);
Samuel Antaobed3c462015-10-02 16:14:20 +00003731
Samuel Antaoee8fb302016-01-06 13:42:12 +00003732 llvm::Function *Fn = nullptr;
3733 llvm::Constant *FnID = nullptr;
Samuel Antaobed3c462015-10-02 16:14:20 +00003734
Samuel Antaobed3c462015-10-02 16:14:20 +00003735 const Expr *IfCond = nullptr;
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +00003736 // Check for the at most one if clause associated with the target region.
3737 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3738 if (C->getNameModifier() == OMPD_unknown ||
3739 C->getNameModifier() == OMPD_target) {
3740 IfCond = C->getCondition();
3741 break;
3742 }
Samuel Antaobed3c462015-10-02 16:14:20 +00003743 }
3744
3745 // Check if we have any device clause associated with the directive.
3746 const Expr *Device = nullptr;
3747 if (auto *C = S.getSingleClause<OMPDeviceClause>()) {
3748 Device = C->getDevice();
3749 }
3750
Samuel Antaoee8fb302016-01-06 13:42:12 +00003751 // Check if we have an if clause whose conditional always evaluates to false
3752 // or if we do not have any targets specified. If so the target region is not
3753 // an offload entry point.
3754 bool IsOffloadEntry = true;
3755 if (IfCond) {
3756 bool Val;
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003757 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
Samuel Antaoee8fb302016-01-06 13:42:12 +00003758 IsOffloadEntry = false;
3759 }
3760 if (CGM.getLangOpts().OMPTargetTriples.empty())
3761 IsOffloadEntry = false;
3762
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003763 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
Samuel Antaoee8fb302016-01-06 13:42:12 +00003764 StringRef ParentName;
3765 // In case we have Ctors/Dtors we use the complete type variant to produce
3766 // the mangling of the device outlined kernel.
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003767 if (auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
Samuel Antaoee8fb302016-01-06 13:42:12 +00003768 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003769 else if (auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
Samuel Antaoee8fb302016-01-06 13:42:12 +00003770 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3771 else
3772 ParentName =
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003773 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
Samuel Antaoee8fb302016-01-06 13:42:12 +00003774
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003775 // Emit target region as a standalone region.
3776 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
3777 IsOffloadEntry, CodeGen);
3778 OMPLexicalScope Scope(CGF, S);
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +00003779 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3780 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003781 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
Samuel Antaobed3c462015-10-02 16:14:20 +00003782 CapturedVars);
Alexey Bataev0bd520b2014-09-19 08:19:49 +00003783}
3784
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00003785static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
3786 PrePostActionTy &Action) {
3787 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3788 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3789 CGF.EmitOMPPrivateClause(S, PrivateScope);
3790 (void)PrivateScope.Privatize();
3791
3792 Action.Enter(CGF);
3793 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3794}
3795
3796void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
3797 StringRef ParentName,
3798 const OMPTargetDirective &S) {
3799 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3800 emitTargetRegion(CGF, S, Action);
3801 };
3802 llvm::Function *Fn;
3803 llvm::Constant *Addr;
3804 // Emit target region as a standalone region.
3805 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3806 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3807 assert(Fn && Addr && "Target device function emission failed.");
3808}
3809
3810void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
3811 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3812 emitTargetRegion(CGF, S, Action);
3813 };
3814 emitCommonOMPTargetDirective(*this, S, CodeGen);
3815}
3816
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003817static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
3818 const OMPExecutableDirective &S,
3819 OpenMPDirectiveKind InnermostKind,
3820 const RegionCodeGenTy &CodeGen) {
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00003821 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
3822 auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
3823 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
Samuel Antaob68e2db2016-03-03 16:20:23 +00003824
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00003825 const OMPNumTeamsClause *NT = S.getSingleClause<OMPNumTeamsClause>();
3826 const OMPThreadLimitClause *TL = S.getSingleClause<OMPThreadLimitClause>();
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003827 if (NT || TL) {
Carlo Bertollic6872252016-04-04 15:55:02 +00003828 Expr *NumTeams = (NT) ? NT->getNumTeams() : nullptr;
3829 Expr *ThreadLimit = (TL) ? TL->getThreadLimit() : nullptr;
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003830
Carlo Bertollic6872252016-04-04 15:55:02 +00003831 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
3832 S.getLocStart());
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003833 }
3834
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00003835 OMPTeamsScope Scope(CGF, S);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003836 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3837 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003838 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
3839 CapturedVars);
3840}
3841
3842void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
Kelvin Li51336dd2016-12-15 17:55:32 +00003843 // Emit teams region as a standalone region.
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003844 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003845 OMPPrivateScope PrivateScope(CGF);
Carlo Bertolli6ad7b5a2016-03-03 22:09:40 +00003846 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3847 CGF.EmitOMPPrivateClause(S, PrivateScope);
Arpith Chacko Jacobfc711b12017-02-16 16:48:49 +00003848 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003849 (void)PrivateScope.Privatize();
3850 CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
Arpith Chacko Jacobfc711b12017-02-16 16:48:49 +00003851 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00003852 };
Alexey Bataev2139ed62017-11-16 18:20:21 +00003853 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
Arpith Chacko Jacobfc711b12017-02-16 16:48:49 +00003854 emitPostUpdateForReductionClause(
3855 *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
Alexey Bataev13314bf2014-10-09 04:18:56 +00003856}
Alexey Bataev6d4ed052015-07-01 06:57:41 +00003857
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00003858static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
3859 const OMPTargetTeamsDirective &S) {
3860 auto *CS = S.getCapturedStmt(OMPD_teams);
3861 Action.Enter(CGF);
Alexey Bataevf9fc42e2017-11-22 14:25:55 +00003862 // Emit teams region as a standalone region.
3863 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
3864 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3865 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3866 CGF.EmitOMPPrivateClause(S, PrivateScope);
3867 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3868 (void)PrivateScope.Privatize();
3869 Action.Enter(CGF);
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00003870 CGF.EmitStmt(CS->getCapturedStmt());
Alexey Bataevf9fc42e2017-11-22 14:25:55 +00003871 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00003872 };
3873 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
Alexey Bataevf9fc42e2017-11-22 14:25:55 +00003874 emitPostUpdateForReductionClause(
3875 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00003876}
3877
3878void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
3879 CodeGenModule &CGM, StringRef ParentName,
3880 const OMPTargetTeamsDirective &S) {
3881 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3882 emitTargetTeamsRegion(CGF, Action, S);
3883 };
3884 llvm::Function *Fn;
3885 llvm::Constant *Addr;
3886 // Emit target region as a standalone region.
3887 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3888 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3889 assert(Fn && Addr && "Target device function emission failed.");
3890}
3891
3892void CodeGenFunction::EmitOMPTargetTeamsDirective(
3893 const OMPTargetTeamsDirective &S) {
3894 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3895 emitTargetTeamsRegion(CGF, Action, S);
3896 };
3897 emitCommonOMPTargetDirective(*this, S, CodeGen);
3898}
3899
Carlo Bertolliba1487b2017-10-04 14:12:09 +00003900void CodeGenFunction::EmitOMPTeamsDistributeDirective(
3901 const OMPTeamsDistributeDirective &S) {
3902
3903 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3904 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3905 };
3906
3907 // Emit teams region as a standalone region.
3908 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
3909 PrePostActionTy &) {
3910 OMPPrivateScope PrivateScope(CGF);
3911 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3912 (void)PrivateScope.Privatize();
3913 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
3914 CodeGenDistribute);
3915 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3916 };
Alexey Bataev95c6dd42017-11-29 15:14:16 +00003917 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
Carlo Bertolliba1487b2017-10-04 14:12:09 +00003918 emitPostUpdateForReductionClause(*this, S,
3919 [](CodeGenFunction &) { return nullptr; });
3920}
3921
Carlo Bertolli62fae152017-11-20 20:46:39 +00003922void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
3923 const OMPTeamsDistributeParallelForDirective &S) {
3924 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3925 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
3926 S.getDistInc());
3927 };
3928
3929 // Emit teams region as a standalone region.
3930 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
3931 PrePostActionTy &) {
3932 OMPPrivateScope PrivateScope(CGF);
3933 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3934 (void)PrivateScope.Privatize();
Alexey Bataev10a54312017-11-27 16:54:08 +00003935 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
3936 CodeGenDistribute);
Carlo Bertolli62fae152017-11-20 20:46:39 +00003937 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3938 };
3939 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
3940 emitPostUpdateForReductionClause(*this, S,
3941 [](CodeGenFunction &) { return nullptr; });
3942}
3943
Carlo Bertolli56a2aa42017-12-04 20:57:19 +00003944void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
3945 const OMPTeamsDistributeParallelForSimdDirective &S) {
3946 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3947 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
3948 S.getDistInc());
3949 };
3950
3951 // Emit teams region as a standalone region.
3952 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
3953 PrePostActionTy &) {
3954 OMPPrivateScope PrivateScope(CGF);
3955 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3956 (void)PrivateScope.Privatize();
3957 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
3958 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
3959 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
3960 };
3961 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
3962 emitPostUpdateForReductionClause(*this, S,
3963 [](CodeGenFunction &) { return nullptr; });
3964}
3965
Alexey Bataev6d4ed052015-07-01 06:57:41 +00003966void CodeGenFunction::EmitOMPCancellationPointDirective(
3967 const OMPCancellationPointDirective &S) {
Alexey Bataev0f34da12015-07-02 04:17:07 +00003968 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
3969 S.getCancelRegion());
Alexey Bataev6d4ed052015-07-01 06:57:41 +00003970}
3971
Alexey Bataev80909872015-07-02 11:25:17 +00003972void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
Alexey Bataev87933c72015-09-18 08:07:34 +00003973 const Expr *IfCond = nullptr;
3974 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3975 if (C->getNameModifier() == OMPD_unknown ||
3976 C->getNameModifier() == OMPD_cancel) {
3977 IfCond = C->getCondition();
3978 break;
3979 }
3980 }
3981 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00003982 S.getCancelRegion());
Alexey Bataev80909872015-07-02 11:25:17 +00003983}
3984
Alexey Bataev81c7ea02015-07-03 09:56:58 +00003985CodeGenFunction::JumpDest
3986CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
Alexey Bataev957d8562016-11-17 15:12:05 +00003987 if (Kind == OMPD_parallel || Kind == OMPD_task ||
3988 Kind == OMPD_target_parallel)
Alexey Bataev81c7ea02015-07-03 09:56:58 +00003989 return ReturnBlock;
Alexey Bataev25e5b442015-09-15 12:52:43 +00003990 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
Alexey Bataev957d8562016-11-17 15:12:05 +00003991 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
3992 Kind == OMPD_distribute_parallel_for ||
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00003993 Kind == OMPD_target_parallel_for ||
Alexey Bataev16e79882017-11-22 21:12:03 +00003994 Kind == OMPD_teams_distribute_parallel_for ||
3995 Kind == OMPD_target_teams_distribute_parallel_for);
Alexey Bataev957d8562016-11-17 15:12:05 +00003996 return OMPCancelStack.getExitBlock();
Alexey Bataev81c7ea02015-07-03 09:56:58 +00003997}
Michael Wong65f367f2015-07-21 13:44:28 +00003998
Samuel Antaocc10b852016-07-28 14:23:26 +00003999void CodeGenFunction::EmitOMPUseDevicePtrClause(
4000 const OMPClause &NC, OMPPrivateScope &PrivateScope,
4001 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4002 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4003 auto OrigVarIt = C.varlist_begin();
4004 auto InitIt = C.inits().begin();
4005 for (auto PvtVarIt : C.private_copies()) {
4006 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4007 auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4008 auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4009
4010 // In order to identify the right initializer we need to match the
4011 // declaration used by the mapping logic. In some cases we may get
4012 // OMPCapturedExprDecl that refers to the original declaration.
4013 const ValueDecl *MatchingVD = OrigVD;
4014 if (auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4015 // OMPCapturedExprDecl are used to privative fields of the current
4016 // structure.
4017 auto *ME = cast<MemberExpr>(OED->getInit());
4018 assert(isa<CXXThisExpr>(ME->getBase()) &&
4019 "Base should be the current struct!");
4020 MatchingVD = ME->getMemberDecl();
4021 }
4022
4023 // If we don't have information about the current list item, move on to
4024 // the next one.
4025 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4026 if (InitAddrIt == CaptureDeviceAddrMap.end())
4027 continue;
4028
4029 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
4030 // Initialize the temporary initialization variable with the address we
4031 // get from the runtime library. We have to cast the source address
4032 // because it is always a void *. References are materialized in the
4033 // privatization scope, so the initialization here disregards the fact
4034 // the original variable is a reference.
4035 QualType AddrQTy =
4036 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4037 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4038 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4039 setAddrOfLocalVar(InitVD, InitAddr);
4040
4041 // Emit private declaration, it will be initialized by the value we
4042 // declaration we just added to the local declarations map.
4043 EmitDecl(*PvtVD);
4044
4045 // The initialization variables reached its purpose in the emission
4046 // ofthe previous declaration, so we don't need it anymore.
4047 LocalDeclMap.erase(InitVD);
4048
4049 // Return the address of the private variable.
4050 return GetAddrOfLocalVar(PvtVD);
4051 });
4052 assert(IsRegistered && "firstprivate var already registered as private");
4053 // Silence the warning about unused variable.
4054 (void)IsRegistered;
4055
4056 ++OrigVarIt;
4057 ++InitIt;
4058 }
4059}
4060
Michael Wong65f367f2015-07-21 13:44:28 +00004061// Generate the instructions for '#pragma omp target data' directive.
4062void CodeGenFunction::EmitOMPTargetDataDirective(
4063 const OMPTargetDataDirective &S) {
Samuel Antaocc10b852016-07-28 14:23:26 +00004064 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4065
4066 // Create a pre/post action to signal the privatization of the device pointer.
4067 // This action can be replaced by the OpenMP runtime code generation to
4068 // deactivate privatization.
4069 bool PrivatizeDevicePointers = false;
4070 class DevicePointerPrivActionTy : public PrePostActionTy {
4071 bool &PrivatizeDevicePointers;
4072
4073 public:
4074 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4075 : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4076 void Enter(CodeGenFunction &CGF) override {
4077 PrivatizeDevicePointers = true;
4078 }
Samuel Antaodf158d52016-04-27 22:58:19 +00004079 };
Samuel Antaocc10b852016-07-28 14:23:26 +00004080 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4081
4082 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4083 CodeGenFunction &CGF, PrePostActionTy &Action) {
4084 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4085 CGF.EmitStmt(
4086 cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
4087 };
4088
4089 // Codegen that selects wheather to generate the privatization code or not.
4090 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4091 &InnermostCodeGen](CodeGenFunction &CGF,
4092 PrePostActionTy &Action) {
4093 RegionCodeGenTy RCG(InnermostCodeGen);
4094 PrivatizeDevicePointers = false;
4095
4096 // Call the pre-action to change the status of PrivatizeDevicePointers if
4097 // needed.
4098 Action.Enter(CGF);
4099
4100 if (PrivatizeDevicePointers) {
4101 OMPPrivateScope PrivateScope(CGF);
4102 // Emit all instances of the use_device_ptr clause.
4103 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4104 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4105 Info.CaptureDeviceAddrMap);
4106 (void)PrivateScope.Privatize();
4107 RCG(CGF);
4108 } else
4109 RCG(CGF);
4110 };
4111
4112 // Forward the provided action to the privatization codegen.
4113 RegionCodeGenTy PrivRCG(PrivCodeGen);
4114 PrivRCG.setAction(Action);
4115
4116 // Notwithstanding the body of the region is emitted as inlined directive,
4117 // we don't use an inline scope as changes in the references inside the
4118 // region are expected to be visible outside, so we do not privative them.
4119 OMPLexicalScope Scope(CGF, S);
4120 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4121 PrivRCG);
4122 };
4123
4124 RegionCodeGenTy RCG(CodeGen);
Samuel Antaodf158d52016-04-27 22:58:19 +00004125
4126 // If we don't have target devices, don't bother emitting the data mapping
4127 // code.
4128 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
Samuel Antaocc10b852016-07-28 14:23:26 +00004129 RCG(*this);
Samuel Antaodf158d52016-04-27 22:58:19 +00004130 return;
4131 }
4132
4133 // Check if we have any if clause associated with the directive.
4134 const Expr *IfCond = nullptr;
4135 if (auto *C = S.getSingleClause<OMPIfClause>())
4136 IfCond = C->getCondition();
4137
4138 // Check if we have any device clause associated with the directive.
4139 const Expr *Device = nullptr;
4140 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4141 Device = C->getDevice();
4142
Samuel Antaocc10b852016-07-28 14:23:26 +00004143 // Set the action to signal privatization of device pointers.
4144 RCG.setAction(PrivAction);
4145
4146 // Emit region code.
4147 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4148 Info);
Michael Wong65f367f2015-07-21 13:44:28 +00004149}
Alexey Bataev49f6e782015-12-01 04:18:41 +00004150
Samuel Antaodf67fc42016-01-19 19:15:56 +00004151void CodeGenFunction::EmitOMPTargetEnterDataDirective(
4152 const OMPTargetEnterDataDirective &S) {
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00004153 // If we don't have target devices, don't bother emitting the data mapping
4154 // code.
4155 if (CGM.getLangOpts().OMPTargetTriples.empty())
4156 return;
4157
4158 // Check if we have any if clause associated with the directive.
4159 const Expr *IfCond = nullptr;
4160 if (auto *C = S.getSingleClause<OMPIfClause>())
4161 IfCond = C->getCondition();
4162
4163 // Check if we have any device clause associated with the directive.
4164 const Expr *Device = nullptr;
4165 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4166 Device = C->getDevice();
4167
Alexey Bataev7828b252017-11-21 17:08:48 +00004168 auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
4169 PrePostActionTy &) {
4170 CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
4171 Device);
4172 };
4173 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4174 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_enter_data,
4175 CodeGen);
Samuel Antaodf67fc42016-01-19 19:15:56 +00004176}
4177
Samuel Antao72590762016-01-19 20:04:50 +00004178void CodeGenFunction::EmitOMPTargetExitDataDirective(
4179 const OMPTargetExitDataDirective &S) {
Samuel Antao8dd66282016-04-27 23:14:30 +00004180 // If we don't have target devices, don't bother emitting the data mapping
4181 // code.
4182 if (CGM.getLangOpts().OMPTargetTriples.empty())
4183 return;
4184
4185 // Check if we have any if clause associated with the directive.
4186 const Expr *IfCond = nullptr;
4187 if (auto *C = S.getSingleClause<OMPIfClause>())
4188 IfCond = C->getCondition();
4189
4190 // Check if we have any device clause associated with the directive.
4191 const Expr *Device = nullptr;
4192 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4193 Device = C->getDevice();
4194
Alexey Bataev7828b252017-11-21 17:08:48 +00004195 auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
4196 PrePostActionTy &) {
4197 CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
4198 Device);
4199 };
4200 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4201 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_exit_data,
4202 CodeGen);
Samuel Antao72590762016-01-19 20:04:50 +00004203}
4204
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004205static void emitTargetParallelRegion(CodeGenFunction &CGF,
4206 const OMPTargetParallelDirective &S,
4207 PrePostActionTy &Action) {
4208 // Get the captured statement associated with the 'parallel' region.
4209 auto *CS = S.getCapturedStmt(OMPD_parallel);
4210 Action.Enter(CGF);
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004211 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &) {
4212 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4213 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4214 CGF.EmitOMPPrivateClause(S, PrivateScope);
4215 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4216 (void)PrivateScope.Privatize();
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004217 // TODO: Add support for clauses.
4218 CGF.EmitStmt(CS->getCapturedStmt());
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004219 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004220 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00004221 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4222 emitEmptyBoundParameters);
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004223 emitPostUpdateForReductionClause(
4224 CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004225}
4226
4227void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
4228 CodeGenModule &CGM, StringRef ParentName,
4229 const OMPTargetParallelDirective &S) {
4230 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4231 emitTargetParallelRegion(CGF, S, Action);
4232 };
4233 llvm::Function *Fn;
4234 llvm::Constant *Addr;
4235 // Emit target region as a standalone region.
4236 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4237 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4238 assert(Fn && Addr && "Target device function emission failed.");
4239}
4240
Arpith Chacko Jacobe955b3d2016-01-26 18:48:41 +00004241void CodeGenFunction::EmitOMPTargetParallelDirective(
4242 const OMPTargetParallelDirective &S) {
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004243 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4244 emitTargetParallelRegion(CGF, S, Action);
4245 };
4246 emitCommonOMPTargetDirective(*this, S, CodeGen);
Arpith Chacko Jacobe955b3d2016-01-26 18:48:41 +00004247}
4248
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00004249static void emitTargetParallelForRegion(CodeGenFunction &CGF,
4250 const OMPTargetParallelForDirective &S,
4251 PrePostActionTy &Action) {
4252 Action.Enter(CGF);
4253 // Emit directive as a combined directive that consists of two implicit
4254 // directives: 'parallel' with 'for' directive.
4255 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev2139ed62017-11-16 18:20:21 +00004256 CodeGenFunction::OMPCancelStackRAII CancelRegion(
4257 CGF, OMPD_target_parallel_for, S.hasCancel());
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00004258 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4259 emitDispatchForLoopBounds);
4260 };
4261 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4262 emitEmptyBoundParameters);
4263}
4264
4265void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
4266 CodeGenModule &CGM, StringRef ParentName,
4267 const OMPTargetParallelForDirective &S) {
4268 // Emit SPMD target parallel for region as a standalone region.
4269 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4270 emitTargetParallelForRegion(CGF, S, Action);
4271 };
4272 llvm::Function *Fn;
4273 llvm::Constant *Addr;
4274 // Emit target region as a standalone region.
4275 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4276 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4277 assert(Fn && Addr && "Target device function emission failed.");
4278}
4279
Arpith Chacko Jacob05bebb52016-02-03 15:46:42 +00004280void CodeGenFunction::EmitOMPTargetParallelForDirective(
4281 const OMPTargetParallelForDirective &S) {
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00004282 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4283 emitTargetParallelForRegion(CGF, S, Action);
4284 };
4285 emitCommonOMPTargetDirective(*this, S, CodeGen);
Arpith Chacko Jacob05bebb52016-02-03 15:46:42 +00004286}
4287
Alexey Bataev5d7edca2017-11-09 17:32:15 +00004288static void
4289emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
4290 const OMPTargetParallelForSimdDirective &S,
4291 PrePostActionTy &Action) {
4292 Action.Enter(CGF);
4293 // Emit directive as a combined directive that consists of two implicit
4294 // directives: 'parallel' with 'for' directive.
4295 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4296 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4297 emitDispatchForLoopBounds);
4298 };
4299 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4300 emitEmptyBoundParameters);
4301}
4302
4303void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
4304 CodeGenModule &CGM, StringRef ParentName,
4305 const OMPTargetParallelForSimdDirective &S) {
4306 // Emit SPMD target parallel for region as a standalone region.
4307 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4308 emitTargetParallelForSimdRegion(CGF, S, Action);
4309 };
4310 llvm::Function *Fn;
4311 llvm::Constant *Addr;
4312 // Emit target region as a standalone region.
4313 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4314 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4315 assert(Fn && Addr && "Target device function emission failed.");
4316}
4317
4318void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
4319 const OMPTargetParallelForSimdDirective &S) {
4320 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4321 emitTargetParallelForSimdRegion(CGF, S, Action);
4322 };
4323 emitCommonOMPTargetDirective(*this, S, CodeGen);
4324}
4325
Alexey Bataev7292c292016-04-25 12:22:29 +00004326/// Emit a helper variable and return corresponding lvalue.
4327static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4328 const ImplicitParamDecl *PVD,
4329 CodeGenFunction::OMPPrivateScope &Privates) {
4330 auto *VDecl = cast<VarDecl>(Helper->getDecl());
4331 Privates.addPrivate(
4332 VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); });
4333}
4334
4335void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
4336 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
4337 // Emit outlined function for task construct.
4338 auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
4339 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
4340 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4341 const Expr *IfCond = nullptr;
4342 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4343 if (C->getNameModifier() == OMPD_unknown ||
4344 C->getNameModifier() == OMPD_taskloop) {
4345 IfCond = C->getCondition();
4346 break;
4347 }
4348 }
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004349
4350 OMPTaskDataTy Data;
4351 // Check if taskloop must be emitted without taskgroup.
4352 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
Alexey Bataev7292c292016-04-25 12:22:29 +00004353 // TODO: Check if we should emit tied or untied task.
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004354 Data.Tied = true;
4355 // Set scheduling for taskloop
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00004356 if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4357 // grainsize clause
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004358 Data.Schedule.setInt(/*IntVal=*/false);
4359 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00004360 } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4361 // num_tasks clause
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004362 Data.Schedule.setInt(/*IntVal=*/true);
4363 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00004364 }
Alexey Bataev7292c292016-04-25 12:22:29 +00004365
4366 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4367 // if (PreCond) {
4368 // for (IV in 0..LastIteration) BODY;
4369 // <Final counter/linear vars updates>;
4370 // }
4371 //
4372
4373 // Emit: if (PreCond) - begin.
4374 // If the condition constant folds and can be elided, avoid emitting the
4375 // whole loop.
4376 bool CondConstant;
4377 llvm::BasicBlock *ContBlock = nullptr;
4378 OMPLoopScope PreInitScope(CGF, S);
4379 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4380 if (!CondConstant)
4381 return;
4382 } else {
4383 auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4384 ContBlock = CGF.createBasicBlock("taskloop.if.end");
4385 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4386 CGF.getProfileCount(&S));
4387 CGF.EmitBlock(ThenBlock);
4388 CGF.incrementProfileCounter(&S);
4389 }
4390
Alexey Bataev1e73ef32016-04-28 12:14:51 +00004391 if (isOpenMPSimdDirective(S.getDirectiveKind()))
4392 CGF.EmitOMPSimdInit(S);
4393
Alexey Bataev7292c292016-04-25 12:22:29 +00004394 OMPPrivateScope LoopScope(CGF);
4395 // Emit helper vars inits.
4396 enum { LowerBound = 5, UpperBound, Stride, LastIter };
4397 auto *I = CS->getCapturedDecl()->param_begin();
4398 auto *LBP = std::next(I, LowerBound);
4399 auto *UBP = std::next(I, UpperBound);
4400 auto *STP = std::next(I, Stride);
4401 auto *LIP = std::next(I, LastIter);
4402 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4403 LoopScope);
4404 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4405 LoopScope);
4406 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4407 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4408 LoopScope);
4409 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
Alexey Bataevf93095a2016-05-05 08:46:22 +00004410 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
Alexey Bataev7292c292016-04-25 12:22:29 +00004411 (void)LoopScope.Privatize();
4412 // Emit the loop iteration variable.
4413 const Expr *IVExpr = S.getIterationVariable();
4414 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4415 CGF.EmitVarDecl(*IVDecl);
4416 CGF.EmitIgnoredExpr(S.getInit());
4417
4418 // Emit the iterations count variable.
4419 // If it is not a variable, Sema decided to calculate iterations count on
4420 // each iteration (e.g., it is foldable into a constant).
4421 if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4422 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4423 // Emit calculation of the iterations count.
4424 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4425 }
4426
4427 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4428 S.getInc(),
4429 [&S](CodeGenFunction &CGF) {
4430 CGF.EmitOMPLoopBody(S, JumpDest());
4431 CGF.EmitStopPoint(&S);
4432 },
4433 [](CodeGenFunction &) {});
4434 // Emit: if (PreCond) - end.
4435 if (ContBlock) {
4436 CGF.EmitBranch(ContBlock);
4437 CGF.EmitBlock(ContBlock, true);
4438 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00004439 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4440 if (HasLastprivateClause) {
4441 CGF.EmitOMPLastprivateClauseFinal(
4442 S, isOpenMPSimdDirective(S.getDirectiveKind()),
4443 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4444 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4445 (*LIP)->getType(), S.getLocStart())));
4446 }
Alexey Bataev7292c292016-04-25 12:22:29 +00004447 };
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004448 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4449 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4450 const OMPTaskDataTy &Data) {
Alexey Bataev7292c292016-04-25 12:22:29 +00004451 auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) {
4452 OMPLoopScope PreInitScope(CGF, S);
Alexey Bataev24b5bae2016-04-28 09:23:51 +00004453 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
4454 OutlinedFn, SharedsTy,
4455 CapturedStruct, IfCond, Data);
Alexey Bataev7292c292016-04-25 12:22:29 +00004456 };
4457 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4458 CodeGen);
4459 };
Alexey Bataev33446032017-07-12 18:09:32 +00004460 if (Data.Nogroup)
4461 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4462 else {
4463 CGM.getOpenMPRuntime().emitTaskgroupRegion(
4464 *this,
4465 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4466 PrePostActionTy &Action) {
4467 Action.Enter(CGF);
4468 CGF.EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4469 },
4470 S.getLocStart());
4471 }
Alexey Bataev7292c292016-04-25 12:22:29 +00004472}
4473
Alexey Bataev49f6e782015-12-01 04:18:41 +00004474void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
Alexey Bataev7292c292016-04-25 12:22:29 +00004475 EmitOMPTaskLoopBasedDirective(S);
Alexey Bataev49f6e782015-12-01 04:18:41 +00004476}
4477
Alexey Bataev0a6ed842015-12-03 09:40:15 +00004478void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
4479 const OMPTaskLoopSimdDirective &S) {
Alexey Bataev1e73ef32016-04-28 12:14:51 +00004480 EmitOMPTaskLoopBasedDirective(S);
Alexey Bataev0a6ed842015-12-03 09:40:15 +00004481}
Samuel Antao686c70c2016-05-26 17:30:50 +00004482
4483// Generate the instructions for '#pragma omp target update' directive.
4484void CodeGenFunction::EmitOMPTargetUpdateDirective(
4485 const OMPTargetUpdateDirective &S) {
Samuel Antao8d2d7302016-05-26 18:30:22 +00004486 // If we don't have target devices, don't bother emitting the data mapping
4487 // code.
4488 if (CGM.getLangOpts().OMPTargetTriples.empty())
4489 return;
4490
4491 // Check if we have any if clause associated with the directive.
4492 const Expr *IfCond = nullptr;
4493 if (auto *C = S.getSingleClause<OMPIfClause>())
4494 IfCond = C->getCondition();
4495
4496 // Check if we have any device clause associated with the directive.
4497 const Expr *Device = nullptr;
4498 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4499 Device = C->getDevice();
4500
Alexey Bataev7828b252017-11-21 17:08:48 +00004501 auto &&CodeGen = [&S, IfCond, Device](CodeGenFunction &CGF,
4502 PrePostActionTy &) {
4503 CGF.CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(CGF, S, IfCond,
4504 Device);
4505 };
4506 OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4507 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_target_update,
4508 CodeGen);
Samuel Antao686c70c2016-05-26 17:30:50 +00004509}