blob: 8369fc236df4d295ac0dea9d3c692de398882769 [file] [log] [blame]
Alexey Bataev9959db52014-05-06 10:08:46 +00001//===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Alexey Bataev9959db52014-05-06 10:08:46 +00006//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit OpenMP nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
Alexey Bataev3392d762016-02-16 11:18:12 +000013#include "CGCleanup.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000014#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
Chandler Carruth0d9593d2015-01-14 11:29:14 +000017#include "TargetInfo.h"
Alexey Bataev61205822019-12-04 09:50:21 -050018#include "clang/AST/ASTContext.h"
Reid Kleckner98031782019-12-09 16:11:56 -080019#include "clang/AST/Attr.h"
20#include "clang/AST/DeclOpenMP.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000021#include "clang/AST/Stmt.h"
22#include "clang/AST/StmtOpenMP.h"
Alexey Bataev8bbf2e32019-11-04 09:59:11 -050023#include "clang/Basic/PrettyStackTrace.h"
Alexey Bataev9959db52014-05-06 10:08:46 +000024using namespace clang;
25using namespace CodeGen;
26
Alexey Bataev3392d762016-02-16 11:18:12 +000027namespace {
28/// Lexical scope for OpenMP executable constructs, that handles correct codegen
29/// for captured expressions.
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000030class OMPLexicalScope : public CodeGenFunction::LexicalScope {
Alexey Bataev3392d762016-02-16 11:18:12 +000031 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
32 for (const auto *C : S.clauses()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +000033 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
34 if (const auto *PreInit =
35 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
Alexey Bataev2bbf7212016-03-03 03:52:24 +000036 for (const auto *I : PreInit->decls()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +000037 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
Alexey Bataev2bbf7212016-03-03 03:52:24 +000038 CGF.EmitVarDecl(cast<VarDecl>(*I));
Alexey Bataevddf3db92018-04-13 17:31:06 +000039 } else {
Alexey Bataev2bbf7212016-03-03 03:52:24 +000040 CodeGenFunction::AutoVarEmission Emission =
41 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
42 CGF.EmitAutoVarCleanups(Emission);
43 }
44 }
Alexey Bataev3392d762016-02-16 11:18:12 +000045 }
46 }
47 }
48 }
Alexey Bataev4ba78a42016-04-27 07:56:03 +000049 CodeGenFunction::OMPPrivateScope InlinedShareds;
50
51 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
52 return CGF.LambdaCaptureFields.lookup(VD) ||
53 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
54 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
55 }
Alexey Bataev3392d762016-02-16 11:18:12 +000056
Alexey Bataev3392d762016-02-16 11:18:12 +000057public:
Alexey Bataev475a7442018-01-12 19:39:11 +000058 OMPLexicalScope(
59 CodeGenFunction &CGF, const OMPExecutableDirective &S,
60 const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
61 const bool EmitPreInitStmt = true)
Alexey Bataev4ba78a42016-04-27 07:56:03 +000062 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
63 InlinedShareds(CGF) {
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000064 if (EmitPreInitStmt)
65 emitPreInitStmt(CGF, S);
Alexey Bataev475a7442018-01-12 19:39:11 +000066 if (!CapturedRegion.hasValue())
67 return;
68 assert(S.hasAssociatedStmt() &&
69 "Expected associated statement for inlined directive.");
70 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
Alexey Bataevddf3db92018-04-13 17:31:06 +000071 for (const auto &C : CS->captures()) {
Alexey Bataev475a7442018-01-12 19:39:11 +000072 if (C.capturesVariable() || C.capturesVariableByCopy()) {
73 auto *VD = C.getCapturedVar();
74 assert(VD == VD->getCanonicalDecl() &&
75 "Canonical decl must be captured.");
76 DeclRefExpr DRE(
Bruno Ricci5fc4db72018-12-21 14:10:18 +000077 CGF.getContext(), const_cast<VarDecl *>(VD),
Alexey Bataev475a7442018-01-12 19:39:11 +000078 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
79 InlinedShareds.isGlobalVarCaptured(VD)),
Alexey Bataeva9b9cc02018-01-23 18:12:38 +000080 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
Alexey Bataev475a7442018-01-12 19:39:11 +000081 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
Akira Hatanakaf139ae32019-12-03 15:17:01 -080082 return CGF.EmitLValue(&DRE).getAddress(CGF);
Alexey Bataev475a7442018-01-12 19:39:11 +000083 });
Alexey Bataev4ba78a42016-04-27 07:56:03 +000084 }
85 }
Alexey Bataev475a7442018-01-12 19:39:11 +000086 (void)InlinedShareds.Privatize();
Alexey Bataev3392d762016-02-16 11:18:12 +000087 }
88};
Alexey Bataev14fa1c62016-03-29 05:34:15 +000089
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000090/// Lexical scope for OpenMP parallel construct, that handles correct codegen
91/// for captured expressions.
92class OMPParallelScope final : public OMPLexicalScope {
93 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
94 OpenMPDirectiveKind Kind = S.getDirectiveKind();
Carlo Bertollib0ff0a62017-04-25 17:52:12 +000095 return !(isOpenMPTargetExecutionDirective(Kind) ||
96 isOpenMPLoopBoundSharingDirective(Kind)) &&
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +000097 isOpenMPParallelDirective(Kind);
98 }
99
100public:
101 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Alexey Bataev475a7442018-01-12 19:39:11 +0000102 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
103 EmitPreInitStmt(S)) {}
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +0000104};
105
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +0000106/// Lexical scope for OpenMP teams construct, that handles correct codegen
107/// for captured expressions.
108class OMPTeamsScope final : public OMPLexicalScope {
109 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110 OpenMPDirectiveKind Kind = S.getDirectiveKind();
111 return !isOpenMPTargetExecutionDirective(Kind) &&
112 isOpenMPTeamsDirective(Kind);
113 }
114
115public:
116 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Alexey Bataev475a7442018-01-12 19:39:11 +0000117 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
118 EmitPreInitStmt(S)) {}
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +0000119};
120
Alexey Bataev5a3af132016-03-29 08:58:54 +0000121/// Private scope for OpenMP loop-based directives, that supports capturing
122/// of used expression from loop statement.
123class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
124 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
Alexey Bataevab4ea222018-03-07 18:17:06 +0000125 CodeGenFunction::OMPMapVars PreCondVars;
Alexey Bataevf71939c2019-09-18 19:24:07 +0000126 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
Alexey Bataevddf3db92018-04-13 17:31:06 +0000127 for (const auto *E : S.counters()) {
Alexey Bataeve83b3e82017-12-08 20:18:58 +0000128 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
Alexey Bataevf71939c2019-09-18 19:24:07 +0000129 EmittedAsPrivate.insert(VD->getCanonicalDecl());
Alexey Bataevab4ea222018-03-07 18:17:06 +0000130 (void)PreCondVars.setVarAddr(
131 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
Alexey Bataeve83b3e82017-12-08 20:18:58 +0000132 }
Alexey Bataevf71939c2019-09-18 19:24:07 +0000133 // Mark private vars as undefs.
134 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
135 for (const Expr *IRef : C->varlists()) {
136 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
137 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
138 (void)PreCondVars.setVarAddr(
139 CGF, OrigVD,
140 Address(llvm::UndefValue::get(
141 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(
142 OrigVD->getType().getNonReferenceType()))),
143 CGF.getContext().getDeclAlign(OrigVD)));
144 }
145 }
146 }
Alexey Bataevab4ea222018-03-07 18:17:06 +0000147 (void)PreCondVars.apply(CGF);
Alexey Bataevbef93a92019-10-07 18:54:57 +0000148 // Emit init, __range and __end variables for C++ range loops.
149 const Stmt *Body =
150 S.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
151 for (unsigned Cnt = 0; Cnt < S.getCollapsedNumber(); ++Cnt) {
Alexey Bataev8bbf2e32019-11-04 09:59:11 -0500152 Body = OMPLoopDirective::tryToFindNextInnerLoop(
153 Body, /*TryImperfectlyNestedLoops=*/true);
Alexey Bataevbef93a92019-10-07 18:54:57 +0000154 if (auto *For = dyn_cast<ForStmt>(Body)) {
155 Body = For->getBody();
156 } else {
157 assert(isa<CXXForRangeStmt>(Body) &&
Alexey Bataevd457f7e2019-10-07 19:57:40 +0000158 "Expected canonical for loop or range-based for loop.");
Alexey Bataevbef93a92019-10-07 18:54:57 +0000159 auto *CXXFor = cast<CXXForRangeStmt>(Body);
160 if (const Stmt *Init = CXXFor->getInit())
161 CGF.EmitStmt(Init);
162 CGF.EmitStmt(CXXFor->getRangeStmt());
163 CGF.EmitStmt(CXXFor->getEndStmt());
164 Body = CXXFor->getBody();
165 }
166 }
Alexey Bataevddf3db92018-04-13 17:31:06 +0000167 if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
George Burgess IV00f70bd2018-03-01 05:43:23 +0000168 for (const auto *I : PreInits->decls())
169 CGF.EmitVarDecl(cast<VarDecl>(*I));
Alexey Bataev5a3af132016-03-29 08:58:54 +0000170 }
Alexey Bataevab4ea222018-03-07 18:17:06 +0000171 PreCondVars.restore(CGF);
Alexey Bataev5a3af132016-03-29 08:58:54 +0000172 }
173
174public:
175 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
176 : CodeGenFunction::RunCleanupsScope(CGF) {
177 emitPreInitStmt(CGF, S);
178 }
179};
180
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000181class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
182 CodeGenFunction::OMPPrivateScope InlinedShareds;
183
184 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
185 return CGF.LambdaCaptureFields.lookup(VD) ||
186 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
187 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
188 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
189 }
190
191public:
192 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
193 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
194 InlinedShareds(CGF) {
195 for (const auto *C : S.clauses()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000196 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
197 if (const auto *PreInit =
198 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000199 for (const auto *I : PreInit->decls()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000200 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000201 CGF.EmitVarDecl(cast<VarDecl>(*I));
Alexey Bataevddf3db92018-04-13 17:31:06 +0000202 } else {
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000203 CodeGenFunction::AutoVarEmission Emission =
204 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
205 CGF.EmitAutoVarCleanups(Emission);
206 }
207 }
208 }
209 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
210 for (const Expr *E : UDP->varlists()) {
211 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
212 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
213 CGF.EmitVarDecl(*OED);
214 }
215 }
216 }
217 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
218 CGF.EmitOMPPrivateClause(S, InlinedShareds);
219 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
220 if (const Expr *E = TG->getReductionRef())
221 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
222 }
223 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
224 while (CS) {
225 for (auto &C : CS->captures()) {
226 if (C.capturesVariable() || C.capturesVariableByCopy()) {
227 auto *VD = C.getCapturedVar();
228 assert(VD == VD->getCanonicalDecl() &&
229 "Canonical decl must be captured.");
Bruno Ricci5fc4db72018-12-21 14:10:18 +0000230 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000231 isCapturedVar(CGF, VD) ||
232 (CGF.CapturedStmtInfo &&
233 InlinedShareds.isGlobalVarCaptured(VD)),
234 VD->getType().getNonReferenceType(), VK_LValue,
Alexey Bataeva9b9cc02018-01-23 18:12:38 +0000235 C.getLocation());
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000236 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800237 return CGF.EmitLValue(&DRE).getAddress(CGF);
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000238 });
239 }
240 }
241 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
242 }
243 (void)InlinedShareds.Privatize();
244 }
245};
246
Alexey Bataev3392d762016-02-16 11:18:12 +0000247} // namespace
248
Alexey Bataevf8365372017-11-17 17:57:25 +0000249static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
250 const OMPExecutableDirective &S,
251 const RegionCodeGenTy &CodeGen);
252
Alexey Bataevf47c4b42017-09-26 13:47:31 +0000253LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000254 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
255 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
Alexey Bataevf47c4b42017-09-26 13:47:31 +0000256 OrigVD = OrigVD->getCanonicalDecl();
257 bool IsCaptured =
258 LambdaCaptureFields.lookup(OrigVD) ||
259 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
260 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
Bruno Ricci5fc4db72018-12-21 14:10:18 +0000261 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
Alexey Bataevf47c4b42017-09-26 13:47:31 +0000262 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
263 return EmitLValue(&DRE);
264 }
265 }
266 return EmitLValue(E);
267}
268
Alexey Bataev1189bd02016-01-26 12:20:39 +0000269llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000270 ASTContext &C = getContext();
Alexey Bataev1189bd02016-01-26 12:20:39 +0000271 llvm::Value *Size = nullptr;
272 auto SizeInChars = C.getTypeSizeInChars(Ty);
273 if (SizeInChars.isZero()) {
274 // getTypeSizeInChars() returns 0 for a VLA.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000275 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
276 VlaSizePair VlaSize = getVLASize(VAT);
Sander de Smalen891af03a2018-02-03 13:55:59 +0000277 Ty = VlaSize.Type;
278 Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
279 : VlaSize.NumElts;
Alexey Bataev1189bd02016-01-26 12:20:39 +0000280 }
281 SizeInChars = C.getTypeSizeInChars(Ty);
282 if (SizeInChars.isZero())
283 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
Alexey Bataevddf3db92018-04-13 17:31:06 +0000284 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
285 }
286 return CGM.getSize(SizeInChars);
Alexey Bataev1189bd02016-01-26 12:20:39 +0000287}
288
Alexey Bataev2377fe92015-09-10 08:12:02 +0000289void CodeGenFunction::GenerateOpenMPCapturedVars(
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000290 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
Alexey Bataev2377fe92015-09-10 08:12:02 +0000291 const RecordDecl *RD = S.getCapturedRecordDecl();
292 auto CurField = RD->field_begin();
293 auto CurCap = S.captures().begin();
294 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
295 E = S.capture_init_end();
296 I != E; ++I, ++CurField, ++CurCap) {
297 if (CurField->hasCapturedVLAType()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000298 const VariableArrayType *VAT = CurField->getCapturedVLAType();
299 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
Samuel Antaobed3c462015-10-02 16:14:20 +0000300 CapturedVars.push_back(Val);
Alexey Bataevddf3db92018-04-13 17:31:06 +0000301 } else if (CurCap->capturesThis()) {
Alexey Bataev2377fe92015-09-10 08:12:02 +0000302 CapturedVars.push_back(CXXThisValue);
Alexey Bataevddf3db92018-04-13 17:31:06 +0000303 } else if (CurCap->capturesVariableByCopy()) {
Alexey Bataev1e491372018-01-23 18:44:14 +0000304 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
Samuel Antao6d004262016-06-16 18:39:34 +0000305
306 // If the field is not a pointer, we need to save the actual value
307 // and load it as a void pointer.
308 if (!CurField->getType()->isAnyPointerType()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000309 ASTContext &Ctx = getContext();
310 Address DstAddr = CreateMemTemp(
Samuel Antao6d004262016-06-16 18:39:34 +0000311 Ctx.getUIntPtrType(),
Alexey Bataevddf3db92018-04-13 17:31:06 +0000312 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
Samuel Antao6d004262016-06-16 18:39:34 +0000313 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
314
Alexey Bataevddf3db92018-04-13 17:31:06 +0000315 llvm::Value *SrcAddrVal = EmitScalarConversion(
Samuel Antao6d004262016-06-16 18:39:34 +0000316 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
Alexey Bataeva9b9cc02018-01-23 18:12:38 +0000317 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
Samuel Antao6d004262016-06-16 18:39:34 +0000318 LValue SrcLV =
319 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
320
321 // Store the value using the source type pointer.
322 EmitStoreThroughLValue(RValue::get(CV), SrcLV);
323
324 // Load the value using the destination type pointer.
Alexey Bataev1e491372018-01-23 18:44:14 +0000325 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
Samuel Antao6d004262016-06-16 18:39:34 +0000326 }
327 CapturedVars.push_back(CV);
328 } else {
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000329 assert(CurCap->capturesVariable() && "Expected capture by reference.");
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800330 CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000331 }
Alexey Bataev2377fe92015-09-10 08:12:02 +0000332 }
333}
334
Alexey Bataeva9b9cc02018-01-23 18:12:38 +0000335static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
336 QualType DstType, StringRef Name,
Alexey Bataev06e80f62019-05-23 18:19:54 +0000337 LValue AddrLV) {
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000338 ASTContext &Ctx = CGF.getContext();
339
Alexey Bataevddf3db92018-04-13 17:31:06 +0000340 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800341 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
Alexey Bataevddf3db92018-04-13 17:31:06 +0000342 Ctx.getPointerType(DstType), Loc);
343 Address TmpAddr =
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000344 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800345 .getAddress(CGF);
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000346 return TmpAddr;
347}
348
Alexey Bataevf7ce1662017-04-10 19:16:45 +0000349static QualType getCanonicalParamType(ASTContext &C, QualType T) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000350 if (T->isLValueReferenceType())
Alexey Bataevf7ce1662017-04-10 19:16:45 +0000351 return C.getLValueReferenceType(
352 getCanonicalParamType(C, T.getNonReferenceType()),
353 /*SpelledAsLValue=*/false);
Alexey Bataevf7ce1662017-04-10 19:16:45 +0000354 if (T->isPointerType())
355 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
Alexey Bataevddf3db92018-04-13 17:31:06 +0000356 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
357 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
Alexey Bataev1b48c5e2017-10-24 19:52:31 +0000358 return getCanonicalParamType(C, VLA->getElementType());
Alexey Bataevddf3db92018-04-13 17:31:06 +0000359 if (!A->isVariablyModifiedType())
Alexey Bataev1b48c5e2017-10-24 19:52:31 +0000360 return C.getCanonicalType(T);
361 }
Alexey Bataevf7ce1662017-04-10 19:16:45 +0000362 return C.getCanonicalParamType(T);
363}
364
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000365namespace {
366 /// Contains required data for proper outlined function codegen.
367 struct FunctionOptions {
368 /// Captured statement for which the function is generated.
369 const CapturedStmt *S = nullptr;
370 /// true if cast to/from UIntPtr is required for variables captured by
371 /// value.
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000372 const bool UIntPtrCastRequired = true;
Alexey Bataeve754b182017-08-09 19:38:53 +0000373 /// true if only casted arguments must be registered as local args or VLA
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000374 /// sizes.
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000375 const bool RegisterCastedArgsOnly = false;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000376 /// Name of the generated function.
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000377 const StringRef FunctionName;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000378 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
379 bool RegisterCastedArgsOnly,
Alexey Bataev4aa19052017-08-08 16:45:36 +0000380 StringRef FunctionName)
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000381 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
382 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
Alexey Bataev4aa19052017-08-08 16:45:36 +0000383 FunctionName(FunctionName) {}
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000384 };
385}
386
Alexey Bataeve754b182017-08-09 19:38:53 +0000387static llvm::Function *emitOutlinedFunctionPrologue(
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000388 CodeGenFunction &CGF, FunctionArgList &Args,
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000389 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000390 &LocalAddrs,
391 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
392 &VLASizes,
393 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
394 const CapturedDecl *CD = FO.S->getCapturedDecl();
395 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000396 assert(CD->hasBody() && "missing CapturedDecl body");
397
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000398 CXXThisValue = nullptr;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000399 // Build the argument list.
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000400 CodeGenModule &CGM = CGF.CGM;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000401 ASTContext &Ctx = CGM.getContext();
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000402 FunctionArgList TargetArgs;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000403 Args.append(CD->param_begin(),
404 std::next(CD->param_begin(), CD->getContextParamPosition()));
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000405 TargetArgs.append(
406 CD->param_begin(),
407 std::next(CD->param_begin(), CD->getContextParamPosition()));
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000408 auto I = FO.S->captures().begin();
Alexey Bataevb45d43c2017-11-22 16:02:03 +0000409 FunctionDecl *DebugFunctionDecl = nullptr;
410 if (!FO.UIntPtrCastRequired) {
411 FunctionProtoType::ExtProtoInfo EPI;
Jonas Devlieghere64a26302018-11-11 00:56:15 +0000412 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
Alexey Bataevb45d43c2017-11-22 16:02:03 +0000413 DebugFunctionDecl = FunctionDecl::Create(
Stephen Kellyf2ceec42018-08-09 21:08:08 +0000414 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
Jonas Devlieghere64a26302018-11-11 00:56:15 +0000415 SourceLocation(), DeclarationName(), FunctionTy,
416 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
417 /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
Alexey Bataevb45d43c2017-11-22 16:02:03 +0000418 }
Alexey Bataevddf3db92018-04-13 17:31:06 +0000419 for (const FieldDecl *FD : RD->fields()) {
Alexey Bataev2377fe92015-09-10 08:12:02 +0000420 QualType ArgType = FD->getType();
421 IdentifierInfo *II = nullptr;
422 VarDecl *CapVar = nullptr;
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000423
424 // If this is a capture by copy and the type is not a pointer, the outlined
425 // function argument type should be uintptr and the value properly casted to
426 // uintptr. This is necessary given that the runtime library is only able to
427 // deal with pointers. We can pass in the same way the VLA type sizes to the
428 // outlined function.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000429 if (FO.UIntPtrCastRequired &&
430 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
431 I->capturesVariableArrayType()))
432 ArgType = Ctx.getUIntPtrType();
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000433
434 if (I->capturesVariable() || I->capturesVariableByCopy()) {
Alexey Bataev2377fe92015-09-10 08:12:02 +0000435 CapVar = I->getCapturedVar();
436 II = CapVar->getIdentifier();
Alexey Bataevddf3db92018-04-13 17:31:06 +0000437 } else if (I->capturesThis()) {
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000438 II = &Ctx.Idents.get("this");
Alexey Bataevddf3db92018-04-13 17:31:06 +0000439 } else {
Alexey Bataev2377fe92015-09-10 08:12:02 +0000440 assert(I->capturesVariableArrayType());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000441 II = &Ctx.Idents.get("vla");
Alexey Bataev2377fe92015-09-10 08:12:02 +0000442 }
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000443 if (ArgType->isVariablyModifiedType())
Alexey Bataev1b48c5e2017-10-24 19:52:31 +0000444 ArgType = getCanonicalParamType(Ctx, ArgType);
Alexey Bataevb45d43c2017-11-22 16:02:03 +0000445 VarDecl *Arg;
446 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
447 Arg = ParmVarDecl::Create(
448 Ctx, DebugFunctionDecl,
Stephen Kellyf2ceec42018-08-09 21:08:08 +0000449 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
Alexey Bataevb45d43c2017-11-22 16:02:03 +0000450 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
451 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
452 } else {
453 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
454 II, ArgType, ImplicitParamDecl::Other);
455 }
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000456 Args.emplace_back(Arg);
457 // Do not cast arguments if we emit function with non-original types.
458 TargetArgs.emplace_back(
459 FO.UIntPtrCastRequired
460 ? Arg
461 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
Alexey Bataev2377fe92015-09-10 08:12:02 +0000462 ++I;
463 }
464 Args.append(
465 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
466 CD->param_end());
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000467 TargetArgs.append(
468 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
469 CD->param_end());
Alexey Bataev2377fe92015-09-10 08:12:02 +0000470
471 // Create the function declaration.
Alexey Bataev2377fe92015-09-10 08:12:02 +0000472 const CGFunctionInfo &FuncInfo =
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000473 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000474 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
475
Alexey Bataevddf3db92018-04-13 17:31:06 +0000476 auto *F =
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000477 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
478 FO.FunctionName, &CGM.getModule());
Alexey Bataev2377fe92015-09-10 08:12:02 +0000479 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
480 if (CD->isNothrow())
Alexey Bataev2c7eee52017-08-04 19:10:54 +0000481 F->setDoesNotThrow();
Alexey Bataevc0f879b2018-04-10 20:10:53 +0000482 F->setDoesNotRecurse();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000483
484 // Generate the function.
Alexey Bataev6e01dc12017-08-14 16:03:47 +0000485 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
Stephen Kellyf2ceec42018-08-09 21:08:08 +0000486 FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
Alexey Bataev2377fe92015-09-10 08:12:02 +0000487 unsigned Cnt = CD->getContextParamPosition();
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000488 I = FO.S->captures().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +0000489 for (const FieldDecl *FD : RD->fields()) {
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000490 // Do not map arguments if we emit function with non-original types.
491 Address LocalAddr(Address::invalid());
492 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
493 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
494 TargetArgs[Cnt]);
495 } else {
496 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
497 }
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000498 // If we are capturing a pointer by copy we don't need to do anything, just
499 // use the value that we get from the arguments.
500 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
Samuel Antao403ffd42016-07-27 22:49:49 +0000501 const VarDecl *CurVD = I->getCapturedVar();
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000502 if (!FO.RegisterCastedArgsOnly)
503 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
Richard Trieucc3949d2016-02-18 22:34:54 +0000504 ++Cnt;
505 ++I;
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000506 continue;
507 }
508
Ivan A. Kosarev5f8c0ca2017-10-10 09:39:32 +0000509 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
510 AlignmentSource::Decl);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000511 if (FD->hasCapturedVLAType()) {
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000512 if (FO.UIntPtrCastRequired) {
Alexey Bataeva9b9cc02018-01-23 18:12:38 +0000513 ArgLVal = CGF.MakeAddrLValue(
514 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
515 Args[Cnt]->getName(), ArgLVal),
516 FD->getType(), AlignmentSource::Decl);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000517 }
Alexey Bataevddf3db92018-04-13 17:31:06 +0000518 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
519 const VariableArrayType *VAT = FD->getCapturedVLAType();
520 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000521 } else if (I->capturesVariable()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000522 const VarDecl *Var = I->getCapturedVar();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000523 QualType VarTy = Var->getType();
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800524 Address ArgAddr = ArgLVal.getAddress(CGF);
Alexey Bataev06e80f62019-05-23 18:19:54 +0000525 if (ArgLVal.getType()->isLValueReferenceType()) {
526 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
527 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
528 assert(ArgLVal.getType()->isPointerType());
529 ArgAddr = CGF.EmitLoadOfPointer(
530 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
Alexey Bataev2377fe92015-09-10 08:12:02 +0000531 }
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000532 if (!FO.RegisterCastedArgsOnly) {
533 LocalAddrs.insert(
534 {Args[Cnt],
535 {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
536 }
Samuel Antao4af1b7b2015-12-02 17:44:43 +0000537 } else if (I->capturesVariableByCopy()) {
538 assert(!FD->getType()->isAnyPointerType() &&
539 "Not expecting a captured pointer.");
Alexey Bataevddf3db92018-04-13 17:31:06 +0000540 const VarDecl *Var = I->getCapturedVar();
Alexey Bataev06e80f62019-05-23 18:19:54 +0000541 LocalAddrs.insert({Args[Cnt],
542 {Var, FO.UIntPtrCastRequired
543 ? castValueFromUintptr(
544 CGF, I->getLocation(), FD->getType(),
545 Args[Cnt]->getName(), ArgLVal)
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800546 : ArgLVal.getAddress(CGF)}});
Alexey Bataev2377fe92015-09-10 08:12:02 +0000547 } else {
548 // If 'this' is captured, load it into CXXThisValue.
549 assert(I->capturesThis());
Alexey Bataev1e491372018-01-23 18:44:14 +0000550 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800551 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}});
Alexey Bataev2377fe92015-09-10 08:12:02 +0000552 }
Richard Trieucc3949d2016-02-18 22:34:54 +0000553 ++Cnt;
554 ++I;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000555 }
556
Alexey Bataeve754b182017-08-09 19:38:53 +0000557 return F;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000558}
559
560llvm::Function *
561CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
562 assert(
563 CapturedStmtInfo &&
564 "CapturedStmtInfo should be set when generating the captured function");
565 const CapturedDecl *CD = S.getCapturedDecl();
566 // Build the argument list.
567 bool NeedWrapperFunction =
568 getDebugInfo() &&
569 CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
570 FunctionArgList Args;
Alexey Bataev3b8d5582017-08-08 18:04:06 +0000571 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000572 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
Alexey Bataeve754b182017-08-09 19:38:53 +0000573 SmallString<256> Buffer;
574 llvm::raw_svector_ostream Out(Buffer);
575 Out << CapturedStmtInfo->getHelperName();
576 if (NeedWrapperFunction)
577 Out << "_debug__";
Alexey Bataev4aa19052017-08-08 16:45:36 +0000578 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
Alexey Bataeve754b182017-08-09 19:38:53 +0000579 Out.str());
580 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
581 VLASizes, CXXThisValue, FO);
Alexey Bataev06e80f62019-05-23 18:19:54 +0000582 CodeGenFunction::OMPPrivateScope LocalScope(*this);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000583 for (const auto &LocalAddrPair : LocalAddrs) {
584 if (LocalAddrPair.second.first) {
Alexey Bataev06e80f62019-05-23 18:19:54 +0000585 LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
586 return LocalAddrPair.second.second;
587 });
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000588 }
589 }
Alexey Bataev06e80f62019-05-23 18:19:54 +0000590 (void)LocalScope.Privatize();
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000591 for (const auto &VLASizePair : VLASizes)
592 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
Serge Pavlov3a561452015-12-06 14:32:39 +0000593 PGO.assignRegionCounters(GlobalDecl(CD), F);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000594 CapturedStmtInfo->EmitBody(*this, CD->getBody());
Alexey Bataev06e80f62019-05-23 18:19:54 +0000595 (void)LocalScope.ForceCleanup();
Alexey Bataev2377fe92015-09-10 08:12:02 +0000596 FinishFunction(CD->getBodyRBrace());
Alexey Bataeve754b182017-08-09 19:38:53 +0000597 if (!NeedWrapperFunction)
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000598 return F;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000599
Alexey Bataevefd884d2017-08-04 21:26:25 +0000600 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
Alexey Bataeve754b182017-08-09 19:38:53 +0000601 /*RegisterCastedArgsOnly=*/true,
602 CapturedStmtInfo->getHelperName());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000603 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
Gheorghe-Teodor Bercead3dcf2f2018-03-14 14:17:45 +0000604 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000605 Args.clear();
606 LocalAddrs.clear();
607 VLASizes.clear();
608 llvm::Function *WrapperF =
609 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
Alexey Bataeve754b182017-08-09 19:38:53 +0000610 WrapperCGF.CXXThisValue, WrapperFO);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000611 llvm::SmallVector<llvm::Value *, 4> CallArgs;
612 for (const auto *Arg : Args) {
613 llvm::Value *CallArg;
614 auto I = LocalAddrs.find(Arg);
615 if (I != LocalAddrs.end()) {
Alexey Bataev7ba57af2017-10-17 16:47:34 +0000616 LValue LV = WrapperCGF.MakeAddrLValue(
617 I->second.second,
618 I->second.first ? I->second.first->getType() : Arg->getType(),
619 AlignmentSource::Decl);
Stephen Kellyf2ceec42018-08-09 21:08:08 +0000620 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000621 } else {
622 auto EI = VLASizes.find(Arg);
Alexey Bataevddf3db92018-04-13 17:31:06 +0000623 if (EI != VLASizes.end()) {
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000624 CallArg = EI->second.second;
Alexey Bataevddf3db92018-04-13 17:31:06 +0000625 } else {
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000626 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
Ivan A. Kosarev5f8c0ca2017-10-10 09:39:32 +0000627 Arg->getType(),
628 AlignmentSource::Decl);
Stephen Kellyf2ceec42018-08-09 21:08:08 +0000629 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000630 }
631 }
Alexey Bataev7ba57af2017-10-17 16:47:34 +0000632 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000633 }
Stephen Kellyf2ceec42018-08-09 21:08:08 +0000634 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
Alexey Bataev3c595a62017-08-14 15:01:03 +0000635 F, CallArgs);
Alexey Bataev1fdfdf72017-06-29 16:43:05 +0000636 WrapperCGF.FinishFunction();
637 return WrapperF;
Alexey Bataev2377fe92015-09-10 08:12:02 +0000638}
639
Alexey Bataev9959db52014-05-06 10:08:46 +0000640//===----------------------------------------------------------------------===//
641// OpenMP Directive Emission
642//===----------------------------------------------------------------------===//
Alexey Bataev420d45b2015-04-14 05:11:24 +0000643void CodeGenFunction::EmitOMPAggregateAssign(
John McCall7f416cc2015-09-08 08:05:57 +0000644 Address DestAddr, Address SrcAddr, QualType OriginalType,
Alexey Bataevddf3db92018-04-13 17:31:06 +0000645 const llvm::function_ref<void(Address, Address)> CopyGen) {
Alexey Bataev420d45b2015-04-14 05:11:24 +0000646 // Perform element-by-element initialization.
647 QualType ElementTy;
John McCall7f416cc2015-09-08 08:05:57 +0000648
649 // Drill down to the base element type on both arrays.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000650 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
651 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
John McCall7f416cc2015-09-08 08:05:57 +0000652 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
653
Alexey Bataevddf3db92018-04-13 17:31:06 +0000654 llvm::Value *SrcBegin = SrcAddr.getPointer();
655 llvm::Value *DestBegin = DestAddr.getPointer();
Alexey Bataev420d45b2015-04-14 05:11:24 +0000656 // Cast from pointer to array type to pointer to single element.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000657 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
Alexey Bataev420d45b2015-04-14 05:11:24 +0000658 // The basic structure here is a while-do loop.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000659 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
660 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
661 llvm::Value *IsEmpty =
Alexey Bataev420d45b2015-04-14 05:11:24 +0000662 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
663 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000664
Alexey Bataev420d45b2015-04-14 05:11:24 +0000665 // Enter the loop body, making that address the current address.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000666 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
Alexey Bataev420d45b2015-04-14 05:11:24 +0000667 EmitBlock(BodyBB);
John McCall7f416cc2015-09-08 08:05:57 +0000668
669 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
670
671 llvm::PHINode *SrcElementPHI =
672 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
673 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
674 Address SrcElementCurrent =
675 Address(SrcElementPHI,
676 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
677
678 llvm::PHINode *DestElementPHI =
679 Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
680 DestElementPHI->addIncoming(DestBegin, EntryBB);
681 Address DestElementCurrent =
682 Address(DestElementPHI,
683 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000684
Alexey Bataev420d45b2015-04-14 05:11:24 +0000685 // Emit copy.
686 CopyGen(DestElementCurrent, SrcElementCurrent);
687
688 // Shift the address forward by one element.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000689 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
John McCall7f416cc2015-09-08 08:05:57 +0000690 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
Alexey Bataevddf3db92018-04-13 17:31:06 +0000691 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
John McCall7f416cc2015-09-08 08:05:57 +0000692 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
Alexey Bataev420d45b2015-04-14 05:11:24 +0000693 // Check whether we've reached the end.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000694 llvm::Value *Done =
Alexey Bataev420d45b2015-04-14 05:11:24 +0000695 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
696 Builder.CreateCondBr(Done, DoneBB, BodyBB);
John McCall7f416cc2015-09-08 08:05:57 +0000697 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
698 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
Alexey Bataev420d45b2015-04-14 05:11:24 +0000699
700 // Done.
701 EmitBlock(DoneBB, /*IsFinished=*/true);
702}
703
John McCall7f416cc2015-09-08 08:05:57 +0000704void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
705 Address SrcAddr, const VarDecl *DestVD,
Alexey Bataev420d45b2015-04-14 05:11:24 +0000706 const VarDecl *SrcVD, const Expr *Copy) {
707 if (OriginalType->isArrayType()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000708 const auto *BO = dyn_cast<BinaryOperator>(Copy);
Alexey Bataev420d45b2015-04-14 05:11:24 +0000709 if (BO && BO->getOpcode() == BO_Assign) {
710 // Perform simple memcpy for simple copying.
Ivan A. Kosarev1860b522018-01-25 14:21:55 +0000711 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
712 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
713 EmitAggregateAssign(Dest, Src, OriginalType);
Alexey Bataev420d45b2015-04-14 05:11:24 +0000714 } else {
715 // For arrays with complex element types perform element by element
716 // copying.
John McCall7f416cc2015-09-08 08:05:57 +0000717 EmitOMPAggregateAssign(
Alexey Bataev420d45b2015-04-14 05:11:24 +0000718 DestAddr, SrcAddr, OriginalType,
John McCall7f416cc2015-09-08 08:05:57 +0000719 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
Alexey Bataev420d45b2015-04-14 05:11:24 +0000720 // Working with the single array element, so have to remap
721 // destination and source variables to corresponding array
722 // elements.
John McCall7f416cc2015-09-08 08:05:57 +0000723 CodeGenFunction::OMPPrivateScope Remap(*this);
Alexey Bataevddf3db92018-04-13 17:31:06 +0000724 Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
725 Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
Alexey Bataev420d45b2015-04-14 05:11:24 +0000726 (void)Remap.Privatize();
John McCall7f416cc2015-09-08 08:05:57 +0000727 EmitIgnoredExpr(Copy);
Alexey Bataev420d45b2015-04-14 05:11:24 +0000728 });
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000729 }
Alexey Bataev420d45b2015-04-14 05:11:24 +0000730 } else {
731 // Remap pseudo source variable to private copy.
John McCall7f416cc2015-09-08 08:05:57 +0000732 CodeGenFunction::OMPPrivateScope Remap(*this);
Alexey Bataevddf3db92018-04-13 17:31:06 +0000733 Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
734 Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
Alexey Bataev420d45b2015-04-14 05:11:24 +0000735 (void)Remap.Privatize();
736 // Emit copying of the whole variable.
John McCall7f416cc2015-09-08 08:05:57 +0000737 EmitIgnoredExpr(Copy);
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000738 }
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000739}
740
Alexey Bataev69c62a92015-04-15 04:52:20 +0000741bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
742 OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000743 if (!HaveInsertPoint())
744 return false;
Alexey Bataev1af5bd52019-03-05 17:47:18 +0000745 bool DeviceConstTarget =
746 getLangOpts().OpenMPIsDevice &&
747 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000748 bool FirstprivateIsLastprivate = false;
749 llvm::DenseSet<const VarDecl *> Lastprivates;
750 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
751 for (const auto *D : C->varlists())
752 Lastprivates.insert(
753 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
754 }
Alexey Bataev69c62a92015-04-15 04:52:20 +0000755 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
Alexey Bataev475a7442018-01-12 19:39:11 +0000756 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
757 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
758 // Force emission of the firstprivate copy if the directive does not emit
759 // outlined function, like omp for, omp simd, omp distribute etc.
760 bool MustEmitFirstprivateCopy =
761 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000762 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000763 auto IRef = C->varlist_begin();
764 auto InitsRef = C->inits().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +0000765 for (const Expr *IInit : C->private_copies()) {
766 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000767 bool ThisFirstprivateIsLastprivate =
768 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
Alexey Bataevddf3db92018-04-13 17:31:06 +0000769 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
Alexey Bataev9c397812019-04-03 17:57:06 +0000770 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
Alexey Bataev475a7442018-01-12 19:39:11 +0000771 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
Alexey Bataev9c397812019-04-03 17:57:06 +0000772 !FD->getType()->isReferenceType() &&
773 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000774 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
775 ++IRef;
776 ++InitsRef;
777 continue;
778 }
Alexey Bataev1af5bd52019-03-05 17:47:18 +0000779 // Do not emit copy for firstprivate constant variables in target regions,
780 // captured by reference.
781 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
Alexey Bataev9c397812019-04-03 17:57:06 +0000782 FD && FD->getType()->isReferenceType() &&
783 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
Alexey Bataev1af5bd52019-03-05 17:47:18 +0000784 (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
785 OrigVD);
786 ++IRef;
787 ++InitsRef;
788 continue;
789 }
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000790 FirstprivateIsLastprivate =
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000791 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000792 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000793 const auto *VDInit =
794 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
Alexey Bataev69c62a92015-04-15 04:52:20 +0000795 bool IsRegistered;
Bruno Ricci5fc4db72018-12-21 14:10:18 +0000796 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
Alexey Bataev7ace49d2016-05-17 08:55:33 +0000797 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
798 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
Alexey Bataeve0ef04f2019-05-23 22:30:43 +0000799 LValue OriginalLVal;
800 if (!FD) {
801 // Check if the firstprivate variable is just a constant value.
802 ConstantEmission CE = tryEmitAsConstant(&DRE);
803 if (CE && !CE.isReference()) {
804 // Constant value, no need to create a copy.
805 ++IRef;
806 ++InitsRef;
807 continue;
808 }
809 if (CE && CE.isReference()) {
810 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
811 } else {
812 assert(!CE && "Expected non-constant firstprivate.");
813 OriginalLVal = EmitLValue(&DRE);
814 }
815 } else {
816 OriginalLVal = EmitLValue(&DRE);
817 }
Alexey Bataevfeddd642016-04-22 09:05:03 +0000818 QualType Type = VD->getType();
Alexey Bataev1d9c15c2015-05-19 12:31:28 +0000819 if (Type->isArrayType()) {
Alexey Bataev69c62a92015-04-15 04:52:20 +0000820 // Emit VarDecl with copy init for arrays.
821 // Get the address of the original variable captured in current
822 // captured region.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000823 IsRegistered = PrivateScope.addPrivate(
824 OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
825 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
826 const Expr *Init = VD->getInit();
827 if (!isa<CXXConstructExpr>(Init) ||
828 isTrivialInitializer(Init)) {
829 // Perform simple memcpy.
830 LValue Dest =
831 MakeAddrLValue(Emission.getAllocatedAddress(), Type);
832 EmitAggregateAssign(Dest, OriginalLVal, Type);
833 } else {
834 EmitOMPAggregateAssign(
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800835 Emission.getAllocatedAddress(),
836 OriginalLVal.getAddress(*this), Type,
Alexey Bataevddf3db92018-04-13 17:31:06 +0000837 [this, VDInit, Init](Address DestElement,
838 Address SrcElement) {
839 // Clean up any temporaries needed by the
840 // initialization.
841 RunCleanupsScope InitScope(*this);
842 // Emit initialization for single element.
843 setAddrOfLocalVar(VDInit, SrcElement);
844 EmitAnyExprToMem(Init, DestElement,
845 Init->getType().getQualifiers(),
846 /*IsInitializer*/ false);
847 LocalDeclMap.erase(VDInit);
848 });
849 }
850 EmitAutoVarCleanups(Emission);
851 return Emission.getAllocatedAddress();
852 });
Alexey Bataev69c62a92015-04-15 04:52:20 +0000853 } else {
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800854 Address OriginalAddr = OriginalLVal.getAddress(*this);
Alexey Bataevddf3db92018-04-13 17:31:06 +0000855 IsRegistered = PrivateScope.addPrivate(
856 OrigVD, [this, VDInit, OriginalAddr, VD]() {
857 // Emit private VarDecl with copy init.
858 // Remap temp VDInit variable to the address of the original
859 // variable (for proper handling of captured global variables).
860 setAddrOfLocalVar(VDInit, OriginalAddr);
861 EmitDecl(*VD);
862 LocalDeclMap.erase(VDInit);
863 return GetAddrOfLocalVar(VD);
864 });
Alexey Bataev69c62a92015-04-15 04:52:20 +0000865 }
866 assert(IsRegistered &&
867 "firstprivate var already registered as private");
868 // Silence the warning about unused variable.
869 (void)IsRegistered;
870 }
Richard Trieucc3949d2016-02-18 22:34:54 +0000871 ++IRef;
872 ++InitsRef;
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000873 }
874 }
Alexey Bataevcd8b6a22016-02-15 08:07:17 +0000875 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
Alexey Bataev4a5bb772014-10-08 14:01:46 +0000876}
877
Alexey Bataev03b340a2014-10-21 03:16:40 +0000878void CodeGenFunction::EmitOMPPrivateClause(
879 const OMPExecutableDirective &D,
880 CodeGenFunction::OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000881 if (!HaveInsertPoint())
882 return;
Alexey Bataev50a64582015-04-22 12:24:45 +0000883 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000884 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
Alexey Bataev03b340a2014-10-21 03:16:40 +0000885 auto IRef = C->varlist_begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +0000886 for (const Expr *IInit : C->private_copies()) {
887 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev50a64582015-04-22 12:24:45 +0000888 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000889 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
890 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
891 // Emit private VarDecl with copy init.
892 EmitDecl(*VD);
893 return GetAddrOfLocalVar(VD);
894 });
Alexey Bataev50a64582015-04-22 12:24:45 +0000895 assert(IsRegistered && "private var already registered as private");
896 // Silence the warning about unused variable.
897 (void)IsRegistered;
898 }
Alexey Bataev03b340a2014-10-21 03:16:40 +0000899 ++IRef;
900 }
901 }
902}
903
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000904bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000905 if (!HaveInsertPoint())
906 return false;
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000907 // threadprivate_var1 = master_threadprivate_var1;
908 // operator=(threadprivate_var2, master_threadprivate_var2);
909 // ...
910 // __kmpc_barrier(&loc, global_tid);
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000911 llvm::DenseSet<const VarDecl *> CopiedVars;
912 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000913 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000914 auto IRef = C->varlist_begin();
915 auto ISrcRef = C->source_exprs().begin();
916 auto IDestRef = C->destination_exprs().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +0000917 for (const Expr *AssignOp : C->assignment_ops()) {
918 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev1d9c15c2015-05-19 12:31:28 +0000919 QualType Type = VD->getType();
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000920 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000921 // Get the address of the master variable. If we are emitting code with
922 // TLS support, the address is passed from the master as field in the
923 // captured declaration.
John McCall7f416cc2015-09-08 08:05:57 +0000924 Address MasterAddr = Address::invalid();
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000925 if (getLangOpts().OpenMPUseTLS &&
926 getContext().getTargetInfo().isTLSSupported()) {
927 assert(CapturedStmtInfo->lookup(VD) &&
928 "Copyin threadprivates should have been captured!");
Bruno Ricci5fc4db72018-12-21 14:10:18 +0000929 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
930 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800931 MasterAddr = EmitLValue(&DRE).getAddress(*this);
Alexey Bataev2377fe92015-09-10 08:12:02 +0000932 LocalDeclMap.erase(VD);
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000933 } else {
John McCall7f416cc2015-09-08 08:05:57 +0000934 MasterAddr =
935 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
936 : CGM.GetAddrOfGlobal(VD),
937 getContext().getDeclAlign(VD));
Samuel Antao9c75cfe2015-07-27 16:38:06 +0000938 }
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000939 // Get the address of the threadprivate variable.
Akira Hatanakaf139ae32019-12-03 15:17:01 -0800940 Address PrivateAddr = EmitLValue(*IRef).getAddress(*this);
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000941 if (CopiedVars.size() == 1) {
942 // At first check if current thread is a master thread. If it is, no
943 // need to copy data.
944 CopyBegin = createBasicBlock("copyin.not.master");
945 CopyEnd = createBasicBlock("copyin.not.master.end");
946 Builder.CreateCondBr(
947 Builder.CreateICmpNE(
John McCall7f416cc2015-09-08 08:05:57 +0000948 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
Alexey Bataevddf3db92018-04-13 17:31:06 +0000949 Builder.CreatePtrToInt(PrivateAddr.getPointer(),
950 CGM.IntPtrTy)),
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000951 CopyBegin, CopyEnd);
952 EmitBlock(CopyBegin);
953 }
Alexey Bataevddf3db92018-04-13 17:31:06 +0000954 const auto *SrcVD =
955 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
956 const auto *DestVD =
957 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
John McCall7f416cc2015-09-08 08:05:57 +0000958 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
Alexey Bataevf56f98c2015-04-16 05:39:01 +0000959 }
960 ++IRef;
961 ++ISrcRef;
962 ++IDestRef;
963 }
964 }
965 if (CopyEnd) {
966 // Exit out of copying procedure for non-master thread.
967 EmitBlock(CopyEnd, /*IsFinished=*/true);
968 return true;
969 }
970 return false;
971}
972
Alexey Bataev38e89532015-04-16 04:54:05 +0000973bool CodeGenFunction::EmitOMPLastprivateClauseInit(
974 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +0000975 if (!HaveInsertPoint())
976 return false;
Alexey Bataev38e89532015-04-16 04:54:05 +0000977 bool HasAtLeastOneLastprivate = false;
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000978 llvm::DenseSet<const VarDecl *> SIMDLCVs;
979 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
Alexey Bataevddf3db92018-04-13 17:31:06 +0000980 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
981 for (const Expr *C : LoopDirective->counters()) {
Alexey Bataev5dff95c2016-04-22 03:56:56 +0000982 SIMDLCVs.insert(
983 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
984 }
985 }
Alexey Bataev38e89532015-04-16 04:54:05 +0000986 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +0000987 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
Alexey Bataevd130fd12015-05-13 10:23:02 +0000988 HasAtLeastOneLastprivate = true;
Alexey Bataeva8a9153a2017-12-29 18:07:07 +0000989 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
990 !getLangOpts().OpenMPSimd)
Alexey Bataevf93095a2016-05-05 08:46:22 +0000991 break;
Alexey Bataev38e89532015-04-16 04:54:05 +0000992 auto IRef = C->varlist_begin();
993 auto IDestRef = C->destination_exprs().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +0000994 for (const Expr *IInit : C->private_copies()) {
Alexey Bataev38e89532015-04-16 04:54:05 +0000995 // Keep the address of the original variable for future update at the end
996 // of the loop.
Alexey Bataevddf3db92018-04-13 17:31:06 +0000997 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataevf93095a2016-05-05 08:46:22 +0000998 // Taskloops do not require additional initialization, it is done in
999 // runtime support library.
Alexey Bataev38e89532015-04-16 04:54:05 +00001000 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001001 const auto *DestVD =
1002 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1003 PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
Bruno Ricci5fc4db72018-12-21 14:10:18 +00001004 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1005 /*RefersToEnclosingVariableOrCapture=*/
1006 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1007 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001008 return EmitLValue(&DRE).getAddress(*this);
Alexey Bataev38e89532015-04-16 04:54:05 +00001009 });
1010 // Check if the variable is also a firstprivate: in this case IInit is
1011 // not generated. Initialization of this variable will happen in codegen
1012 // for 'firstprivate' clause.
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001013 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001014 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1015 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
Alexey Bataevf93095a2016-05-05 08:46:22 +00001016 // Emit private VarDecl with copy init.
1017 EmitDecl(*VD);
1018 return GetAddrOfLocalVar(VD);
1019 });
Alexey Bataevd130fd12015-05-13 10:23:02 +00001020 assert(IsRegistered &&
1021 "lastprivate var already registered as private");
1022 (void)IsRegistered;
1023 }
Alexey Bataev38e89532015-04-16 04:54:05 +00001024 }
Richard Trieucc3949d2016-02-18 22:34:54 +00001025 ++IRef;
1026 ++IDestRef;
Alexey Bataev38e89532015-04-16 04:54:05 +00001027 }
1028 }
1029 return HasAtLeastOneLastprivate;
1030}
1031
1032void CodeGenFunction::EmitOMPLastprivateClauseFinal(
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001033 const OMPExecutableDirective &D, bool NoFinals,
1034 llvm::Value *IsLastIterCond) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001035 if (!HaveInsertPoint())
1036 return;
Alexey Bataev38e89532015-04-16 04:54:05 +00001037 // Emit following code:
1038 // if (<IsLastIterCond>) {
1039 // orig_var1 = private_orig_var1;
1040 // ...
1041 // orig_varn = private_orig_varn;
1042 // }
Alexey Bataevfc087ec2015-06-16 13:14:42 +00001043 llvm::BasicBlock *ThenBB = nullptr;
1044 llvm::BasicBlock *DoneBB = nullptr;
1045 if (IsLastIterCond) {
1046 ThenBB = createBasicBlock(".omp.lastprivate.then");
1047 DoneBB = createBasicBlock(".omp.lastprivate.done");
1048 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1049 EmitBlock(ThenBB);
1050 }
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001051 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1052 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
Alexey Bataevddf3db92018-04-13 17:31:06 +00001053 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001054 auto IC = LoopDirective->counters().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001055 for (const Expr *F : LoopDirective->finals()) {
1056 const auto *D =
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001057 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1058 if (NoFinals)
1059 AlreadyEmittedVars.insert(D);
1060 else
1061 LoopCountersAndUpdates[D] = F;
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001062 ++IC;
Alexey Bataev7a228ff2015-05-21 07:59:51 +00001063 }
1064 }
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001065 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1066 auto IRef = C->varlist_begin();
1067 auto ISrcRef = C->source_exprs().begin();
1068 auto IDestRef = C->destination_exprs().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001069 for (const Expr *AssignOp : C->assignment_ops()) {
1070 const auto *PrivateVD =
1071 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001072 QualType Type = PrivateVD->getType();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001073 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001074 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1075 // If lastprivate variable is a loop control variable for loop-based
1076 // directive, update its value before copyin back to original
1077 // variable.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001078 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001079 EmitIgnoredExpr(FinalExpr);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001080 const auto *SrcVD =
1081 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1082 const auto *DestVD =
1083 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001084 // Get the address of the original variable.
1085 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1086 // Get the address of the private variable.
1087 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001088 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001089 PrivateAddr =
John McCall7f416cc2015-09-08 08:05:57 +00001090 Address(Builder.CreateLoad(PrivateAddr),
1091 getNaturalTypeAlignment(RefTy->getPointeeType()));
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001092 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
Alexey Bataev38e89532015-04-16 04:54:05 +00001093 }
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001094 ++IRef;
1095 ++ISrcRef;
1096 ++IDestRef;
Alexey Bataev38e89532015-04-16 04:54:05 +00001097 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00001098 if (const Expr *PostUpdate = C->getPostUpdateExpr())
Alexey Bataev005248a2016-02-25 05:25:57 +00001099 EmitIgnoredExpr(PostUpdate);
Alexey Bataev38e89532015-04-16 04:54:05 +00001100 }
Alexey Bataev8ffcc942016-02-18 13:48:15 +00001101 if (IsLastIterCond)
Alexey Bataevfc087ec2015-06-16 13:14:42 +00001102 EmitBlock(DoneBB, /*IsFinished=*/true);
Alexey Bataev38e89532015-04-16 04:54:05 +00001103}
1104
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001105void CodeGenFunction::EmitOMPReductionClauseInit(
1106 const OMPExecutableDirective &D,
1107 CodeGenFunction::OMPPrivateScope &PrivateScope) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001108 if (!HaveInsertPoint())
1109 return;
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001110 SmallVector<const Expr *, 4> Shareds;
1111 SmallVector<const Expr *, 4> Privates;
1112 SmallVector<const Expr *, 4> ReductionOps;
1113 SmallVector<const Expr *, 4> LHSs;
1114 SmallVector<const Expr *, 4> RHSs;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001115 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
Alexey Bataevf24e7b12015-10-08 09:10:53 +00001116 auto IPriv = C->privates().begin();
Alexey Bataeva839ddd2016-03-17 10:19:46 +00001117 auto IRed = C->reduction_ops().begin();
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001118 auto ILHS = C->lhs_exprs().begin();
1119 auto IRHS = C->rhs_exprs().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001120 for (const Expr *Ref : C->varlists()) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001121 Shareds.emplace_back(Ref);
1122 Privates.emplace_back(*IPriv);
1123 ReductionOps.emplace_back(*IRed);
1124 LHSs.emplace_back(*ILHS);
1125 RHSs.emplace_back(*IRHS);
1126 std::advance(IPriv, 1);
1127 std::advance(IRed, 1);
1128 std::advance(ILHS, 1);
1129 std::advance(IRHS, 1);
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001130 }
1131 }
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001132 ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1133 unsigned Count = 0;
1134 auto ILHS = LHSs.begin();
1135 auto IRHS = RHSs.begin();
1136 auto IPriv = Privates.begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001137 for (const Expr *IRef : Shareds) {
1138 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001139 // Emit private VarDecl with reduction init.
1140 RedCG.emitSharedLValue(*this, Count);
1141 RedCG.emitAggregateType(*this, Count);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001142 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001143 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1144 RedCG.getSharedLValue(Count),
1145 [&Emission](CodeGenFunction &CGF) {
1146 CGF.EmitAutoVarInit(Emission);
1147 return true;
1148 });
1149 EmitAutoVarCleanups(Emission);
1150 Address BaseAddr = RedCG.adjustPrivateAddress(
1151 *this, Count, Emission.getAllocatedAddress());
1152 bool IsRegistered = PrivateScope.addPrivate(
Alexey Bataevddf3db92018-04-13 17:31:06 +00001153 RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001154 assert(IsRegistered && "private var already registered as private");
1155 // Silence the warning about unused variable.
1156 (void)IsRegistered;
1157
Alexey Bataevddf3db92018-04-13 17:31:06 +00001158 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1159 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
Jonas Hahnfeld4525c822017-10-23 19:01:35 +00001160 QualType Type = PrivateVD->getType();
1161 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1162 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001163 // Store the address of the original variable associated with the LHS
1164 // implicit variable.
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001165 PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
1166 return RedCG.getSharedLValue(Count).getAddress(*this);
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001167 });
Alexey Bataevddf3db92018-04-13 17:31:06 +00001168 PrivateScope.addPrivate(
1169 RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
Jonas Hahnfeld4525c822017-10-23 19:01:35 +00001170 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1171 isa<ArraySubscriptExpr>(IRef)) {
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001172 // Store the address of the original variable associated with the LHS
1173 // implicit variable.
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001174 PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
1175 return RedCG.getSharedLValue(Count).getAddress(*this);
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001176 });
Alexey Bataevddf3db92018-04-13 17:31:06 +00001177 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001178 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1179 ConvertTypeForMem(RHSVD->getType()),
1180 "rhs.begin");
1181 });
1182 } else {
1183 QualType Type = PrivateVD->getType();
1184 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001185 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this);
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001186 // Store the address of the original variable associated with the LHS
1187 // implicit variable.
1188 if (IsArray) {
1189 OriginalAddr = Builder.CreateElementBitCast(
1190 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1191 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00001192 PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001193 PrivateScope.addPrivate(
Alexey Bataevddf3db92018-04-13 17:31:06 +00001194 RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
Alexey Bataev5c40bec2017-07-13 13:36:14 +00001195 return IsArray
1196 ? Builder.CreateElementBitCast(
1197 GetAddrOfLocalVar(PrivateVD),
1198 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1199 : GetAddrOfLocalVar(PrivateVD);
1200 });
1201 }
1202 ++ILHS;
1203 ++IRHS;
1204 ++IPriv;
1205 ++Count;
1206 }
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001207}
1208
1209void CodeGenFunction::EmitOMPReductionClauseFinal(
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001210 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001211 if (!HaveInsertPoint())
1212 return;
Alexey Bataevf24e7b12015-10-08 09:10:53 +00001213 llvm::SmallVector<const Expr *, 8> Privates;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001214 llvm::SmallVector<const Expr *, 8> LHSExprs;
1215 llvm::SmallVector<const Expr *, 8> RHSExprs;
1216 llvm::SmallVector<const Expr *, 8> ReductionOps;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001217 bool HasAtLeastOneReduction = false;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001218 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001219 HasAtLeastOneReduction = true;
Alexey Bataevf24e7b12015-10-08 09:10:53 +00001220 Privates.append(C->privates().begin(), C->privates().end());
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001221 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1222 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1223 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1224 }
1225 if (HasAtLeastOneReduction) {
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001226 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1227 isOpenMPParallelDirective(D.getDirectiveKind()) ||
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00001228 ReductionKind == OMPD_simd;
1229 bool SimpleReduction = ReductionKind == OMPD_simd;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001230 // Emit nowait reduction if nowait clause is present or directive is a
1231 // parallel directive (it always has implicit barrier).
1232 CGM.getOpenMPRuntime().emitReduction(
Stephen Kelly1c301dc2018-08-09 21:09:38 +00001233 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001234 {WithNowait, SimpleReduction, ReductionKind});
Alexey Bataev794ba0d2015-04-10 10:43:45 +00001235 }
1236}
1237
Alexey Bataev61205072016-03-02 04:57:40 +00001238static void emitPostUpdateForReductionClause(
1239 CodeGenFunction &CGF, const OMPExecutableDirective &D,
Alexey Bataevddf3db92018-04-13 17:31:06 +00001240 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
Alexey Bataev61205072016-03-02 04:57:40 +00001241 if (!CGF.HaveInsertPoint())
1242 return;
1243 llvm::BasicBlock *DoneBB = nullptr;
1244 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001245 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
Alexey Bataev61205072016-03-02 04:57:40 +00001246 if (!DoneBB) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001247 if (llvm::Value *Cond = CondGen(CGF)) {
Alexey Bataev61205072016-03-02 04:57:40 +00001248 // If the first post-update expression is found, emit conditional
1249 // block if it was requested.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001250 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
Alexey Bataev61205072016-03-02 04:57:40 +00001251 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1252 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1253 CGF.EmitBlock(ThenBB);
1254 }
1255 }
1256 CGF.EmitIgnoredExpr(PostUpdate);
1257 }
1258 }
1259 if (DoneBB)
1260 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1261}
1262
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001263namespace {
1264/// Codegen lambda for appending distribute lower and upper bounds to outlined
1265/// parallel function. This is necessary for combined constructs such as
1266/// 'distribute parallel for'
1267typedef llvm::function_ref<void(CodeGenFunction &,
1268 const OMPExecutableDirective &,
1269 llvm::SmallVectorImpl<llvm::Value *> &)>
1270 CodeGenBoundParametersTy;
1271} // anonymous namespace
1272
1273static void emitCommonOMPParallelDirective(
1274 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1275 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1276 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00001277 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
James Y Knight9871db02019-02-05 16:42:33 +00001278 llvm::Function *OutlinedFn =
Alexey Bataevddf3db92018-04-13 17:31:06 +00001279 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1280 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001281 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
Alexey Bataev1d677132015-04-22 13:57:31 +00001282 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001283 llvm::Value *NumThreads =
1284 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1285 /*IgnoreResultAssign=*/true);
Alexey Bataev1d677132015-04-22 13:57:31 +00001286 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00001287 CGF, NumThreads, NumThreadsClause->getBeginLoc());
Alexey Bataev1d677132015-04-22 13:57:31 +00001288 }
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001289 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001290 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
Alexey Bataev7f210c62015-06-18 13:40:03 +00001291 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00001292 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
Alexey Bataev7f210c62015-06-18 13:40:03 +00001293 }
Alexey Bataev1d677132015-04-22 13:57:31 +00001294 const Expr *IfCond = nullptr;
Alexey Bataev7371aa32015-09-03 08:45:56 +00001295 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1296 if (C->getNameModifier() == OMPD_unknown ||
1297 C->getNameModifier() == OMPD_parallel) {
1298 IfCond = C->getCondition();
1299 break;
1300 }
Alexey Bataev1d677132015-04-22 13:57:31 +00001301 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001302
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +00001303 OMPParallelScope Scope(CGF, S);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001304 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001305 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1306 // lower and upper bounds with the pragma 'for' chunking mechanism.
1307 // The following lambda takes care of appending the lower and upper bound
1308 // parameters when necessary
1309 CodeGenBoundParameters(CGF, S, CapturedVars);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00001310 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00001311 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
Alexey Bataev2377fe92015-09-10 08:12:02 +00001312 CapturedVars, IfCond);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001313}
1314
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001315static void emitEmptyBoundParameters(CodeGenFunction &,
1316 const OMPExecutableDirective &,
1317 llvm::SmallVectorImpl<llvm::Value *> &) {}
1318
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001319void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001320 // Emit parallel region as a standalone region.
Alexey Bataevc99042b2018-03-15 18:10:54 +00001321 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00001322 Action.Enter(CGF);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001323 OMPPrivateScope PrivateScope(CGF);
Alexey Bataevf56f98c2015-04-16 05:39:01 +00001324 bool Copyins = CGF.EmitOMPCopyinClause(S);
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00001325 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1326 if (Copyins) {
Alexey Bataev69c62a92015-04-15 04:52:20 +00001327 // Emit implicit barrier to synchronize threads and avoid data races on
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00001328 // propagation master's thread values of threadprivate variables to local
1329 // instances of that variables of all other implicit threads.
Alexey Bataev25e5b442015-09-15 12:52:43 +00001330 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00001331 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
Alexey Bataev25e5b442015-09-15 12:52:43 +00001332 /*ForceSimpleCall=*/true);
Alexey Bataev69c62a92015-04-15 04:52:20 +00001333 }
1334 CGF.EmitOMPPrivateClause(S, PrivateScope);
1335 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1336 (void)PrivateScope.Privatize();
Alexey Bataev475a7442018-01-12 19:39:11 +00001337 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00001338 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001339 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001340 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1341 emitEmptyBoundParameters);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001342 emitPostUpdateForReductionClause(*this, S,
1343 [](CodeGenFunction &) { return nullptr; });
Alexey Bataev9959db52014-05-06 10:08:46 +00001344}
Alexander Musman515ad8c2014-05-22 08:54:05 +00001345
Alexey Bataev8bbf2e32019-11-04 09:59:11 -05001346static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1347 int MaxLevel, int Level = 0) {
1348 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1349 const Stmt *SimplifiedS = S->IgnoreContainers();
1350 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1351 PrettyStackTraceLoc CrashInfo(
1352 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1353 "LLVM IR generation of compound statement ('{}')");
1354
1355 // Keep track of the current cleanup stack depth, including debug scopes.
1356 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1357 for (const Stmt *CurStmt : CS->body())
1358 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1359 return;
1360 }
1361 if (SimplifiedS == NextLoop) {
1362 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1363 S = For->getBody();
1364 } else {
1365 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1366 "Expected canonical for loop or range-based for loop.");
1367 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1368 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1369 S = CXXFor->getBody();
1370 }
1371 if (Level + 1 < MaxLevel) {
1372 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
1373 S, /*TryImperfectlyNestedLoops=*/true);
1374 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1375 return;
1376 }
1377 }
1378 CGF.EmitStmt(S);
1379}
1380
Alexey Bataev0f34da12015-07-02 04:17:07 +00001381void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1382 JumpDest LoopExit) {
Alexander Musmana5f070a2014-10-01 06:03:56 +00001383 RunCleanupsScope BodyScope(*this);
1384 // Update counters values on current iteration.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001385 for (const Expr *UE : D.updates())
1386 EmitIgnoredExpr(UE);
Alexander Musman3276a272015-03-21 10:12:56 +00001387 // Update the linear variables.
Alexey Bataev617db5f2017-12-04 15:38:33 +00001388 // In distribute directives only loop counters may be marked as linear, no
1389 // need to generate the code for them.
1390 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1391 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001392 for (const Expr *UE : C->updates())
1393 EmitIgnoredExpr(UE);
Alexey Bataev617db5f2017-12-04 15:38:33 +00001394 }
Alexander Musman3276a272015-03-21 10:12:56 +00001395 }
1396
Alexander Musmana5f070a2014-10-01 06:03:56 +00001397 // On a continue in the body, jump to the end.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001398 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
Alexey Bataev0f34da12015-07-02 04:17:07 +00001399 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
Alexey Bataevf8be4762019-08-14 19:30:06 +00001400 for (const Expr *E : D.finals_conditions()) {
1401 if (!E)
1402 continue;
1403 // Check that loop counter in non-rectangular nest fits into the iteration
1404 // space.
1405 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1406 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1407 getProfileCount(D.getBody()));
1408 EmitBlock(NextBB);
1409 }
Alexey Bataevbef93a92019-10-07 18:54:57 +00001410 // Emit loop variables for C++ range loops.
1411 const Stmt *Body =
1412 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
Alexander Musmana5f070a2014-10-01 06:03:56 +00001413 // Emit loop body.
Alexey Bataev8bbf2e32019-11-04 09:59:11 -05001414 emitBody(*this, Body,
1415 OMPLoopDirective::tryToFindNextInnerLoop(
1416 Body, /*TryImperfectlyNestedLoops=*/true),
1417 D.getCollapsedNumber());
1418
Alexander Musmana5f070a2014-10-01 06:03:56 +00001419 // The end (updates/cleanups).
1420 EmitBlock(Continue.getBlock());
1421 BreakContinueStack.pop_back();
Alexander Musmana5f070a2014-10-01 06:03:56 +00001422}
1423
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001424void CodeGenFunction::EmitOMPInnerLoop(
1425 const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1426 const Expr *IncExpr,
Alexey Bataevddf3db92018-04-13 17:31:06 +00001427 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1428 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
Alexander Musmand196ef22014-10-07 08:57:09 +00001429 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001430
1431 // Start the loop with a block that tests the condition.
Alexander Musmand196ef22014-10-07 08:57:09 +00001432 auto CondBlock = createBasicBlock("omp.inner.for.cond");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001433 EmitBlock(CondBlock);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001434 const SourceRange R = S.getSourceRange();
Amara Emerson652795d2016-11-10 14:44:30 +00001435 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1436 SourceLocToDebugLoc(R.getEnd()));
Alexander Musmana5f070a2014-10-01 06:03:56 +00001437
1438 // If there are any cleanups between here and the loop-exit scope,
1439 // create a block to stage a loop exit along.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001440 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
Alexey Bataev2df54a02015-03-12 08:53:29 +00001441 if (RequiresCleanup)
Alexander Musmand196ef22014-10-07 08:57:09 +00001442 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001443
Alexey Bataevddf3db92018-04-13 17:31:06 +00001444 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001445
Alexey Bataev2df54a02015-03-12 08:53:29 +00001446 // Emit condition.
Justin Bogner66242d62015-04-23 23:06:47 +00001447 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
Alexander Musmana5f070a2014-10-01 06:03:56 +00001448 if (ExitBlock != LoopExit.getBlock()) {
1449 EmitBlock(ExitBlock);
1450 EmitBranchThroughCleanup(LoopExit);
1451 }
1452
1453 EmitBlock(LoopBody);
Justin Bogner66242d62015-04-23 23:06:47 +00001454 incrementProfileCounter(&S);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001455
1456 // Create a block for the increment.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001457 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
Alexander Musmana5f070a2014-10-01 06:03:56 +00001458 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1459
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001460 BodyGen(*this);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001461
1462 // Emit "IV = IV + 1" and a back-edge to the condition block.
1463 EmitBlock(Continue.getBlock());
Alexey Bataev2df54a02015-03-12 08:53:29 +00001464 EmitIgnoredExpr(IncExpr);
Alexey Bataev98eb6e32015-04-22 11:15:40 +00001465 PostIncGen(*this);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001466 BreakContinueStack.pop_back();
1467 EmitBranch(CondBlock);
1468 LoopStack.pop();
1469 // Emit the fall-through block.
1470 EmitBlock(LoopExit.getBlock());
1471}
1472
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001473bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001474 if (!HaveInsertPoint())
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001475 return false;
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001476 // Emit inits for the linear variables.
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001477 bool HasLinears = false;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001478 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001479 for (const Expr *Init : C->inits()) {
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001480 HasLinears = true;
Alexey Bataevddf3db92018-04-13 17:31:06 +00001481 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1482 if (const auto *Ref =
1483 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001484 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001485 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
Bruno Ricci5fc4db72018-12-21 14:10:18 +00001486 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
Alexey Bataevef549a82016-03-09 09:49:09 +00001487 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1488 VD->getInit()->getType(), VK_LValue,
1489 VD->getInit()->getExprLoc());
1490 EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1491 VD->getType()),
1492 /*capturedByInit=*/false);
1493 EmitAutoVarCleanups(Emission);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001494 } else {
Alexey Bataevef549a82016-03-09 09:49:09 +00001495 EmitVarDecl(*VD);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001496 }
Alexander Musmana5f070a2014-10-01 06:03:56 +00001497 }
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001498 // Emit the linear steps for the linear clauses.
1499 // If a step is not constant, it is pre-calculated before the loop.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001500 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1501 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00001502 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001503 // Emit calculation of the linear step.
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00001504 EmitIgnoredExpr(CS);
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001505 }
Alexander Musmana5f070a2014-10-01 06:03:56 +00001506 }
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00001507 return HasLinears;
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001508}
1509
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001510void CodeGenFunction::EmitOMPLinearClauseFinal(
1511 const OMPLoopDirective &D,
Alexey Bataevddf3db92018-04-13 17:31:06 +00001512 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001513 if (!HaveInsertPoint())
Alexey Bataev8ef31412015-12-18 07:58:25 +00001514 return;
Alexey Bataevef549a82016-03-09 09:49:09 +00001515 llvm::BasicBlock *DoneBB = nullptr;
Alexander Musman3276a272015-03-21 10:12:56 +00001516 // Emit the final values of the linear variables.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001517 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
Alexey Bataev39f915b82015-05-08 10:41:21 +00001518 auto IC = C->varlist_begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001519 for (const Expr *F : C->finals()) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001520 if (!DoneBB) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001521 if (llvm::Value *Cond = CondGen(*this)) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001522 // If the first post-update expression is found, emit conditional
1523 // block if it was requested.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001524 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001525 DoneBB = createBasicBlock(".omp.linear.pu.done");
1526 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1527 EmitBlock(ThenBB);
Alexey Bataevef549a82016-03-09 09:49:09 +00001528 }
1529 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00001530 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
Bruno Ricci5fc4db72018-12-21 14:10:18 +00001531 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001532 CapturedStmtInfo->lookup(OrigVD) != nullptr,
Alexey Bataev39f915b82015-05-08 10:41:21 +00001533 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001534 Address OrigAddr = EmitLValue(&DRE).getAddress(*this);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001535 CodeGenFunction::OMPPrivateScope VarScope(*this);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001536 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
Alexey Bataev39f915b82015-05-08 10:41:21 +00001537 (void)VarScope.Privatize();
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001538 EmitIgnoredExpr(F);
Alexey Bataev39f915b82015-05-08 10:41:21 +00001539 ++IC;
Alexander Musman3276a272015-03-21 10:12:56 +00001540 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00001541 if (const Expr *PostUpdate = C->getPostUpdateExpr())
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001542 EmitIgnoredExpr(PostUpdate);
Alexander Musman3276a272015-03-21 10:12:56 +00001543 }
Alexey Bataevef549a82016-03-09 09:49:09 +00001544 if (DoneBB)
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001545 EmitBlock(DoneBB, /*IsFinished=*/true);
Alexander Musmana5f070a2014-10-01 06:03:56 +00001546}
1547
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001548static void emitAlignedClause(CodeGenFunction &CGF,
1549 const OMPExecutableDirective &D) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001550 if (!CGF.HaveInsertPoint())
1551 return;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001552 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
Erich Keanef7593952019-10-11 14:59:44 +00001553 llvm::APInt ClauseAlignment(64, 0);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001554 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1555 auto *AlignmentCI =
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001556 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
Erich Keanef7593952019-10-11 14:59:44 +00001557 ClauseAlignment = AlignmentCI->getValue();
Alexander Musman09184fe2014-09-30 05:29:28 +00001558 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00001559 for (const Expr *E : Clause->varlists()) {
Erich Keanef7593952019-10-11 14:59:44 +00001560 llvm::APInt Alignment(ClauseAlignment);
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001561 if (Alignment == 0) {
1562 // OpenMP [2.8.1, Description]
1563 // If no optional parameter is specified, implementation-defined default
1564 // alignments for SIMD instructions on the target platforms are assumed.
1565 Alignment =
Alexey Bataev00396512015-07-02 03:40:19 +00001566 CGF.getContext()
1567 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1568 E->getType()->getPointeeType()))
1569 .getQuantity();
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001570 }
Erich Keanef7593952019-10-11 14:59:44 +00001571 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001572 "alignment is not power of 2");
1573 if (Alignment != 0) {
1574 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
Roman Lebedevbd1c0872019-01-15 09:44:25 +00001575 CGF.EmitAlignmentAssumption(
Erich Keanef7593952019-10-11 14:59:44 +00001576 PtrValue, E, /*No second loc needed*/ SourceLocation(),
1577 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001578 }
Alexander Musman09184fe2014-09-30 05:29:28 +00001579 }
1580 }
1581}
1582
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001583void CodeGenFunction::EmitOMPPrivateLoopCounters(
1584 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1585 if (!HaveInsertPoint())
Alexey Bataev8ef31412015-12-18 07:58:25 +00001586 return;
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001587 auto I = S.private_counters().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001588 for (const Expr *E : S.counters()) {
1589 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1590 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
Alexey Bataevab4ea222018-03-07 18:17:06 +00001591 // Emit var without initialization.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001592 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
Alexey Bataevab4ea222018-03-07 18:17:06 +00001593 EmitAutoVarCleanups(VarEmission);
1594 LocalDeclMap.erase(PrivateVD);
1595 (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1596 return VarEmission.getAllocatedAddress();
Alexey Bataev435ad7b2014-10-10 09:48:26 +00001597 });
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001598 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1599 VD->hasGlobalStorage()) {
Alexey Bataevab4ea222018-03-07 18:17:06 +00001600 (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
Bruno Ricci5fc4db72018-12-21 14:10:18 +00001601 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001602 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1603 E->getType(), VK_LValue, E->getExprLoc());
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001604 return EmitLValue(&DRE).getAddress(*this);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001605 });
Alexey Bataevab4ea222018-03-07 18:17:06 +00001606 } else {
1607 (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1608 return VarEmission.getAllocatedAddress();
1609 });
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001610 }
Alexey Bataeva8899172015-08-06 12:30:57 +00001611 ++I;
Alexey Bataev435ad7b2014-10-10 09:48:26 +00001612 }
Alexey Bataevf138fda2018-08-13 19:04:24 +00001613 // Privatize extra loop counters used in loops for ordered(n) clauses.
1614 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1615 if (!C->getNumForLoops())
1616 continue;
1617 for (unsigned I = S.getCollapsedNumber(),
1618 E = C->getLoopNumIterations().size();
1619 I < E; ++I) {
Mike Rice0ed46662018-09-20 17:19:41 +00001620 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
Alexey Bataevf138fda2018-08-13 19:04:24 +00001621 const auto *VD = cast<VarDecl>(DRE->getDecl());
Alexey Bataev0d8fcdf2019-03-14 20:36:00 +00001622 // Override only those variables that can be captured to avoid re-emission
1623 // of the variables declared within the loops.
1624 if (DRE->refersToEnclosingVariableOrCapture()) {
Alexey Bataevf138fda2018-08-13 19:04:24 +00001625 (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1626 return CreateMemTemp(DRE->getType(), VD->getName());
1627 });
1628 }
1629 }
1630 }
Alexey Bataev435ad7b2014-10-10 09:48:26 +00001631}
1632
Alexey Bataev62dbb972015-04-22 11:59:37 +00001633static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1634 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1635 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001636 if (!CGF.HaveInsertPoint())
1637 return;
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001638 {
1639 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001640 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001641 (void)PreCondScope.Privatize();
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001642 // Get initial values of real counters.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001643 for (const Expr *I : S.inits()) {
Alexey Bataev6e8248f2015-06-11 10:53:56 +00001644 CGF.EmitIgnoredExpr(I);
1645 }
Alexey Bataev62dbb972015-04-22 11:59:37 +00001646 }
Alexey Bataevf8be4762019-08-14 19:30:06 +00001647 // Create temp loop control variables with their init values to support
1648 // non-rectangular loops.
1649 CodeGenFunction::OMPMapVars PreCondVars;
1650 for (const Expr * E: S.dependent_counters()) {
1651 if (!E)
1652 continue;
1653 assert(!E->getType().getNonReferenceType()->isRecordType() &&
1654 "dependent counter must not be an iterator.");
1655 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1656 Address CounterAddr =
1657 CGF.CreateMemTemp(VD->getType().getNonReferenceType());
1658 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
1659 }
1660 (void)PreCondVars.apply(CGF);
1661 for (const Expr *E : S.dependent_inits()) {
1662 if (!E)
1663 continue;
1664 CGF.EmitIgnoredExpr(E);
1665 }
Alexey Bataev62dbb972015-04-22 11:59:37 +00001666 // Check that loop is executed at least one time.
1667 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
Alexey Bataevf8be4762019-08-14 19:30:06 +00001668 PreCondVars.restore(CGF);
Alexey Bataev62dbb972015-04-22 11:59:37 +00001669}
1670
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001671void CodeGenFunction::EmitOMPLinearClause(
1672 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1673 if (!HaveInsertPoint())
Alexey Bataev8ef31412015-12-18 07:58:25 +00001674 return;
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001675 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1676 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001677 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1678 for (const Expr *C : LoopDirective->counters()) {
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001679 SIMDLCVs.insert(
1680 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1681 }
1682 }
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001683 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
Alexey Bataevbd9fec12015-08-18 06:47:21 +00001684 auto CurPrivate = C->privates().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001685 for (const Expr *E : C->varlists()) {
1686 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1687 const auto *PrivateVD =
Alexey Bataevbd9fec12015-08-18 06:47:21 +00001688 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001689 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001690 bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001691 // Emit private VarDecl with copy init.
1692 EmitVarDecl(*PrivateVD);
1693 return GetAddrOfLocalVar(PrivateVD);
1694 });
1695 assert(IsRegistered && "linear var already registered as private");
1696 // Silence the warning about unused variable.
1697 (void)IsRegistered;
Alexey Bataevddf3db92018-04-13 17:31:06 +00001698 } else {
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001699 EmitVarDecl(*PrivateVD);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001700 }
Alexey Bataevbd9fec12015-08-18 06:47:21 +00001701 ++CurPrivate;
Alexander Musman3276a272015-03-21 10:12:56 +00001702 }
1703 }
1704}
1705
Alexey Bataev45bfad52015-08-21 12:19:04 +00001706static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001707 const OMPExecutableDirective &D,
1708 bool IsMonotonic) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001709 if (!CGF.HaveInsertPoint())
1710 return;
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001711 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
Alexey Bataev45bfad52015-08-21 12:19:04 +00001712 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1713 /*ignoreResult=*/true);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001714 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
Alexey Bataev45bfad52015-08-21 12:19:04 +00001715 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1716 // In presence of finite 'safelen', it may be unsafe to mark all
1717 // the memory instructions parallel, because loop-carried
1718 // dependences of 'safelen' iterations are possible.
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001719 if (!IsMonotonic)
1720 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00001721 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001722 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1723 /*ignoreResult=*/true);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001724 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
Tyler Nowickida46d0e2015-07-14 23:03:09 +00001725 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001726 // In presence of finite 'safelen', it may be unsafe to mark all
1727 // the memory instructions parallel, because loop-carried
1728 // dependences of 'safelen' iterations are possible.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001729 CGF.LoopStack.setParallel(/*Enable=*/false);
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00001730 }
1731}
1732
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001733void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1734 bool IsMonotonic) {
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001735 // Walk clauses and process safelen/lastprivate.
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001736 LoopStack.setParallel(!IsMonotonic);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001737 LoopStack.setVectorizeEnable();
Alexey Bataeva6f2a142015-12-31 06:52:34 +00001738 emitSimdlenSafelenClause(*this, D, IsMonotonic);
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001739}
1740
Alexey Bataevef549a82016-03-09 09:49:09 +00001741void CodeGenFunction::EmitOMPSimdFinal(
1742 const OMPLoopDirective &D,
Alexey Bataevddf3db92018-04-13 17:31:06 +00001743 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
Alexey Bataev8ef31412015-12-18 07:58:25 +00001744 if (!HaveInsertPoint())
1745 return;
Alexey Bataevef549a82016-03-09 09:49:09 +00001746 llvm::BasicBlock *DoneBB = nullptr;
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001747 auto IC = D.counters().begin();
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001748 auto IPC = D.private_counters().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001749 for (const Expr *F : D.finals()) {
1750 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1751 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1752 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001753 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1754 OrigVD->hasGlobalStorage() || CED) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001755 if (!DoneBB) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001756 if (llvm::Value *Cond = CondGen(*this)) {
Alexey Bataevef549a82016-03-09 09:49:09 +00001757 // If the first post-update expression is found, emit conditional
1758 // block if it was requested.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001759 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
Alexey Bataevef549a82016-03-09 09:49:09 +00001760 DoneBB = createBasicBlock(".omp.final.done");
1761 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1762 EmitBlock(ThenBB);
1763 }
1764 }
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001765 Address OrigAddr = Address::invalid();
Alexey Bataevab4ea222018-03-07 18:17:06 +00001766 if (CED) {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001767 OrigAddr =
1768 EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this);
Alexey Bataevab4ea222018-03-07 18:17:06 +00001769 } else {
Bruno Ricci5fc4db72018-12-21 14:10:18 +00001770 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001771 /*RefersToEnclosingVariableOrCapture=*/false,
1772 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
Akira Hatanakaf139ae32019-12-03 15:17:01 -08001773 OrigAddr = EmitLValue(&DRE).getAddress(*this);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001774 }
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001775 OMPPrivateScope VarScope(*this);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001776 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001777 (void)VarScope.Privatize();
1778 EmitIgnoredExpr(F);
1779 }
1780 ++IC;
Alexey Bataev5dff95c2016-04-22 03:56:56 +00001781 ++IPC;
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001782 }
Alexey Bataevef549a82016-03-09 09:49:09 +00001783 if (DoneBB)
1784 EmitBlock(DoneBB, /*IsFinished=*/true);
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00001785}
1786
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001787static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1788 const OMPLoopDirective &S,
1789 CodeGenFunction::JumpDest LoopExit) {
1790 CGF.EmitOMPLoopBody(S, LoopExit);
1791 CGF.EmitStopPoint(&S);
Hans Wennborged129ae2017-04-27 17:02:25 +00001792}
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001793
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00001794/// Emit a helper variable and return corresponding lvalue.
1795static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1796 const DeclRefExpr *Helper) {
1797 auto VDecl = cast<VarDecl>(Helper->getDecl());
1798 CGF.EmitVarDecl(*VDecl);
1799 return CGF.EmitLValue(Helper);
1800}
1801
Alexey Bataev103f3c9e2019-11-20 15:59:03 -05001802static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
1803 const RegionCodeGenTy &SimdInitGen,
1804 const RegionCodeGenTy &BodyCodeGen) {
1805 auto &&ThenGen = [&SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
1806 PrePostActionTy &) {
1807 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
1808 SimdInitGen(CGF);
1809
1810 BodyCodeGen(CGF);
1811 };
1812 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
1813 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
1814 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
1815
1816 BodyCodeGen(CGF);
1817 };
1818 const Expr *IfCond = nullptr;
1819 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1820 if (CGF.getLangOpts().OpenMP >= 50 &&
1821 (C->getNameModifier() == OMPD_unknown ||
1822 C->getNameModifier() == OMPD_simd)) {
1823 IfCond = C->getCondition();
1824 break;
1825 }
1826 }
1827 if (IfCond) {
1828 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
1829 } else {
1830 RegionCodeGenTy ThenRCG(ThenGen);
1831 ThenRCG(CGF);
1832 }
1833}
1834
Alexey Bataevf8365372017-11-17 17:57:25 +00001835static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
1836 PrePostActionTy &Action) {
1837 Action.Enter(CGF);
1838 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
1839 "Expected simd directive");
1840 OMPLoopScope PreInitScope(CGF, S);
1841 // if (PreCond) {
1842 // for (IV in 0..LastIteration) BODY;
1843 // <Final counter/linear vars updates>;
1844 // }
1845 //
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00001846 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
1847 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
1848 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
1849 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1850 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1851 }
Alexander Musmana5f070a2014-10-01 06:03:56 +00001852
Alexey Bataevf8365372017-11-17 17:57:25 +00001853 // Emit: if (PreCond) - begin.
1854 // If the condition constant folds and can be elided, avoid emitting the
1855 // whole loop.
1856 bool CondConstant;
1857 llvm::BasicBlock *ContBlock = nullptr;
1858 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1859 if (!CondConstant)
1860 return;
1861 } else {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001862 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
Alexey Bataevf8365372017-11-17 17:57:25 +00001863 ContBlock = CGF.createBasicBlock("simd.if.end");
1864 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1865 CGF.getProfileCount(&S));
1866 CGF.EmitBlock(ThenBlock);
1867 CGF.incrementProfileCounter(&S);
1868 }
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001869
Alexey Bataevf8365372017-11-17 17:57:25 +00001870 // Emit the loop iteration variable.
1871 const Expr *IVExpr = S.getIterationVariable();
Alexey Bataevddf3db92018-04-13 17:31:06 +00001872 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
Alexey Bataevf8365372017-11-17 17:57:25 +00001873 CGF.EmitVarDecl(*IVDecl);
1874 CGF.EmitIgnoredExpr(S.getInit());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001875
Alexey Bataevf8365372017-11-17 17:57:25 +00001876 // Emit the iterations count variable.
1877 // If it is not a variable, Sema decided to calculate iterations count on
1878 // each iteration (e.g., it is foldable into a constant).
Alexey Bataevddf3db92018-04-13 17:31:06 +00001879 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
Alexey Bataevf8365372017-11-17 17:57:25 +00001880 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1881 // Emit calculation of the iterations count.
1882 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1883 }
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001884
Alexey Bataevf8365372017-11-17 17:57:25 +00001885 emitAlignedClause(CGF, S);
1886 (void)CGF.EmitOMPLinearClauseInit(S);
1887 {
1888 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1889 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1890 CGF.EmitOMPLinearClause(S, LoopScope);
1891 CGF.EmitOMPPrivateClause(S, LoopScope);
1892 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1893 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1894 (void)LoopScope.Privatize();
Alexey Bataev60705422018-10-30 15:50:12 +00001895 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
1896 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
Alexey Bataevd08c0562019-11-19 12:07:54 -05001897
Alexey Bataev103f3c9e2019-11-20 15:59:03 -05001898 emitCommonSimdLoop(
1899 CGF, S,
1900 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1901 CGF.EmitOMPSimdInit(S);
1902 },
1903 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
1904 CGF.EmitOMPInnerLoop(
1905 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
1906 [&S](CodeGenFunction &CGF) {
1907 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
1908 CGF.EmitStopPoint(&S);
1909 },
1910 [](CodeGenFunction &) {});
1911 });
Alexey Bataevddf3db92018-04-13 17:31:06 +00001912 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
Alexey Bataevf8365372017-11-17 17:57:25 +00001913 // Emit final copy of the lastprivate variables at the end of loops.
1914 if (HasLastprivateClause)
1915 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1916 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001917 emitPostUpdateForReductionClause(CGF, S,
1918 [](CodeGenFunction &) { return nullptr; });
Alexey Bataevf8365372017-11-17 17:57:25 +00001919 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00001920 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
Alexey Bataevf8365372017-11-17 17:57:25 +00001921 // Emit: if (PreCond) - end.
1922 if (ContBlock) {
1923 CGF.EmitBranch(ContBlock);
1924 CGF.EmitBlock(ContBlock, true);
1925 }
1926}
1927
1928void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
1929 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1930 emitOMPSimdRegion(CGF, S, Action);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00001931 };
Alexey Bataev475a7442018-01-12 19:39:11 +00001932 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00001933 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
Alexander Musman515ad8c2014-05-22 08:54:05 +00001934}
1935
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001936void CodeGenFunction::EmitOMPOuterLoop(
1937 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1938 CodeGenFunction::OMPPrivateScope &LoopScope,
1939 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1940 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1941 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00001942 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
Alexander Musman92bdaab2015-03-12 13:37:50 +00001943
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001944 const Expr *IVExpr = S.getIterationVariable();
1945 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1946 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1947
Alexey Bataevddf3db92018-04-13 17:31:06 +00001948 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001949
1950 // Start the loop with a block that tests the condition.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001951 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001952 EmitBlock(CondBlock);
Alexey Bataevddf3db92018-04-13 17:31:06 +00001953 const SourceRange R = S.getSourceRange();
Amara Emerson652795d2016-11-10 14:44:30 +00001954 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1955 SourceLocToDebugLoc(R.getEnd()));
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001956
1957 llvm::Value *BoolCondVal = nullptr;
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00001958 if (!DynamicOrOrdered) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001959 // UB = min(UB, GlobalUB) or
1960 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1961 // 'distribute parallel for')
1962 EmitIgnoredExpr(LoopArgs.EUB);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001963 // IV = LB
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001964 EmitIgnoredExpr(LoopArgs.Init);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001965 // IV < UB
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001966 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001967 } else {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001968 BoolCondVal =
Stephen Kellyf2ceec42018-08-09 21:08:08 +00001969 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001970 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001971 }
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001972
1973 // If there are any cleanups between here and the loop-exit scope,
1974 // create a block to stage a loop exit along.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001975 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001976 if (LoopScope.requiresCleanups())
1977 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1978
Alexey Bataevddf3db92018-04-13 17:31:06 +00001979 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001980 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1981 if (ExitBlock != LoopExit.getBlock()) {
1982 EmitBlock(ExitBlock);
1983 EmitBranchThroughCleanup(LoopExit);
1984 }
1985 EmitBlock(LoopBody);
1986
Alexander Musman92bdaab2015-03-12 13:37:50 +00001987 // Emit "IV = LB" (in case of static schedule, we have already calculated new
1988 // LB for loop condition and emitted it above).
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00001989 if (DynamicOrOrdered)
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00001990 EmitIgnoredExpr(LoopArgs.Init);
Alexander Musman92bdaab2015-03-12 13:37:50 +00001991
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001992 // Create a block for the increment.
Alexey Bataevddf3db92018-04-13 17:31:06 +00001993 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
Alexander Musmandf7a8e22015-01-22 08:49:35 +00001994 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1995
Alexey Bataev103f3c9e2019-11-20 15:59:03 -05001996 emitCommonSimdLoop(
1997 *this, S,
1998 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
1999 // Generate !llvm.loop.parallel metadata for loads and stores for loops
2000 // with dynamic/guided scheduling and without ordered clause.
2001 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
2002 CGF.LoopStack.setParallel(!IsMonotonic);
2003 else
2004 CGF.EmitOMPSimdInit(S, IsMonotonic);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002005 },
Alexey Bataev103f3c9e2019-11-20 15:59:03 -05002006 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2007 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2008 SourceLocation Loc = S.getBeginLoc();
2009 // when 'distribute' is not combined with a 'for':
2010 // while (idx <= UB) { BODY; ++idx; }
2011 // when 'distribute' is combined with a 'for'
2012 // (e.g. 'distribute parallel for')
2013 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
2014 CGF.EmitOMPInnerLoop(
2015 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2016 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2017 CodeGenLoop(CGF, S, LoopExit);
2018 },
2019 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2020 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2021 });
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002022 });
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002023
2024 EmitBlock(Continue.getBlock());
2025 BreakContinueStack.pop_back();
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002026 if (!DynamicOrOrdered) {
Alexander Musman92bdaab2015-03-12 13:37:50 +00002027 // Emit "LB = LB + Stride", "UB = UB + Stride".
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002028 EmitIgnoredExpr(LoopArgs.NextLB);
2029 EmitIgnoredExpr(LoopArgs.NextUB);
Alexander Musman92bdaab2015-03-12 13:37:50 +00002030 }
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002031
2032 EmitBranch(CondBlock);
2033 LoopStack.pop();
2034 // Emit the fall-through block.
2035 EmitBlock(LoopExit.getBlock());
2036
2037 // Tell the runtime we are done.
Alexey Bataev957d8562016-11-17 15:12:05 +00002038 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
2039 if (!DynamicOrOrdered)
Stephen Kelly1c301dc2018-08-09 21:09:38 +00002040 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
Alexey Bataevf43f7142017-09-06 16:17:35 +00002041 S.getDirectiveKind());
Alexey Bataev957d8562016-11-17 15:12:05 +00002042 };
2043 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002044}
2045
2046void CodeGenFunction::EmitOMPForOuterLoop(
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002047 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002048 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002049 const OMPLoopArguments &LoopArgs,
2050 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002051 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002052
2053 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002054 const bool DynamicOrOrdered =
2055 Ordered || RT.isDynamic(ScheduleKind.Schedule);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002056
2057 assert((Ordered ||
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002058 !RT.isStaticNonchunked(ScheduleKind.Schedule,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002059 LoopArgs.Chunk != nullptr)) &&
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002060 "static non-chunked schedule does not need outer loop");
2061
2062 // Emit outer loop.
2063 //
2064 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2065 // When schedule(dynamic,chunk_size) is specified, the iterations are
2066 // distributed to threads in the team in chunks as the threads request them.
2067 // Each thread executes a chunk of iterations, then requests another chunk,
2068 // until no chunks remain to be distributed. Each chunk contains chunk_size
2069 // iterations, except for the last chunk to be distributed, which may have
2070 // fewer iterations. When no chunk_size is specified, it defaults to 1.
2071 //
2072 // When schedule(guided,chunk_size) is specified, the iterations are assigned
2073 // to threads in the team in chunks as the executing threads request them.
2074 // Each thread executes a chunk of iterations, then requests another chunk,
2075 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
2076 // each chunk is proportional to the number of unassigned iterations divided
2077 // by the number of threads in the team, decreasing to 1. For a chunk_size
2078 // with value k (greater than 1), the size of each chunk is determined in the
2079 // same way, with the restriction that the chunks do not contain fewer than k
2080 // iterations (except for the last chunk to be assigned, which may have fewer
2081 // than k iterations).
2082 //
2083 // When schedule(auto) is specified, the decision regarding scheduling is
2084 // delegated to the compiler and/or runtime system. The programmer gives the
2085 // implementation the freedom to choose any possible mapping of iterations to
2086 // threads in the team.
2087 //
2088 // When schedule(runtime) is specified, the decision regarding scheduling is
2089 // deferred until run time, and the schedule and chunk size are taken from the
2090 // run-sched-var ICV. If the ICV is set to auto, the schedule is
2091 // implementation defined
2092 //
2093 // while(__kmpc_dispatch_next(&LB, &UB)) {
2094 // idx = LB;
2095 // while (idx <= UB) { BODY; ++idx;
2096 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
2097 // } // inner loop
2098 // }
2099 //
2100 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2101 // When schedule(static, chunk_size) is specified, iterations are divided into
2102 // chunks of size chunk_size, and the chunks are assigned to the threads in
2103 // the team in a round-robin fashion in the order of the thread number.
2104 //
2105 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
2106 // while (idx <= UB) { BODY; ++idx; } // inner loop
2107 // LB = LB + ST;
2108 // UB = UB + ST;
2109 // }
2110 //
2111
2112 const Expr *IVExpr = S.getIterationVariable();
2113 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2114 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2115
2116 if (DynamicOrOrdered) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002117 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2118 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002119 llvm::Value *LBVal = DispatchBounds.first;
2120 llvm::Value *UBVal = DispatchBounds.second;
2121 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
2122 LoopArgs.Chunk};
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002123 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002124 IVSigned, Ordered, DipatchRTInputValues);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002125 } else {
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002126 CGOpenMPRuntime::StaticRTInput StaticInit(
2127 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
2128 LoopArgs.ST, LoopArgs.Chunk);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002129 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002130 ScheduleKind, StaticInit);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002131 }
2132
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002133 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
2134 const unsigned IVSize,
2135 const bool IVSigned) {
2136 if (Ordered) {
2137 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
2138 IVSigned);
2139 }
2140 };
2141
2142 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
2143 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
2144 OuterLoopArgs.IncExpr = S.getInc();
2145 OuterLoopArgs.Init = S.getInit();
2146 OuterLoopArgs.Cond = S.getCond();
2147 OuterLoopArgs.NextLB = S.getNextLowerBound();
2148 OuterLoopArgs.NextUB = S.getNextUpperBound();
2149 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
2150 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002151}
2152
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002153static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
2154 const unsigned IVSize, const bool IVSigned) {}
2155
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002156void CodeGenFunction::EmitOMPDistributeOuterLoop(
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002157 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
2158 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
2159 const CodeGenLoopTy &CodeGenLoopContent) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002160
Alexey Bataevddf3db92018-04-13 17:31:06 +00002161 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002162
2163 // Emit outer loop.
2164 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
2165 // dynamic
2166 //
2167
2168 const Expr *IVExpr = S.getIterationVariable();
2169 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2170 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2171
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002172 CGOpenMPRuntime::StaticRTInput StaticInit(
2173 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2174 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002175 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00002176
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002177 // for combined 'distribute' and 'for' the increment expression of distribute
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00002178 // is stored in DistInc. For 'distribute' alone, it is in Inc.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002179 Expr *IncExpr;
2180 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
2181 IncExpr = S.getDistInc();
2182 else
2183 IncExpr = S.getInc();
2184
2185 // this routine is shared by 'omp distribute parallel for' and
2186 // 'omp distribute': select the right EUB expression depending on the
2187 // directive
2188 OMPLoopArguments OuterLoopArgs;
2189 OuterLoopArgs.LB = LoopArgs.LB;
2190 OuterLoopArgs.UB = LoopArgs.UB;
2191 OuterLoopArgs.ST = LoopArgs.ST;
2192 OuterLoopArgs.IL = LoopArgs.IL;
2193 OuterLoopArgs.Chunk = LoopArgs.Chunk;
2194 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2195 ? S.getCombinedEnsureUpperBound()
2196 : S.getEnsureUpperBound();
2197 OuterLoopArgs.IncExpr = IncExpr;
2198 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2199 ? S.getCombinedInit()
2200 : S.getInit();
2201 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2202 ? S.getCombinedCond()
2203 : S.getCond();
2204 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2205 ? S.getCombinedNextLowerBound()
2206 : S.getNextLowerBound();
2207 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2208 ? S.getCombinedNextUpperBound()
2209 : S.getNextUpperBound();
2210
2211 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2212 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2213 emitEmptyOrdered);
2214}
2215
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002216static std::pair<LValue, LValue>
2217emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
2218 const OMPExecutableDirective &S) {
2219 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2220 LValue LB =
2221 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2222 LValue UB =
2223 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2224
2225 // When composing 'distribute' with 'for' (e.g. as in 'distribute
2226 // parallel for') we need to use the 'distribute'
2227 // chunk lower and upper bounds rather than the whole loop iteration
2228 // space. These are parameters to the outlined function for 'parallel'
2229 // and we copy the bounds of the previous schedule into the
2230 // the current ones.
2231 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2232 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00002233 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2234 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002235 PrevLBVal = CGF.EmitScalarConversion(
2236 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00002237 LS.getIterationVariable()->getType(),
2238 LS.getPrevLowerBoundVariable()->getExprLoc());
2239 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2240 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002241 PrevUBVal = CGF.EmitScalarConversion(
2242 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00002243 LS.getIterationVariable()->getType(),
2244 LS.getPrevUpperBoundVariable()->getExprLoc());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002245
2246 CGF.EmitStoreOfScalar(PrevLBVal, LB);
2247 CGF.EmitStoreOfScalar(PrevUBVal, UB);
2248
2249 return {LB, UB};
2250}
2251
2252/// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2253/// we need to use the LB and UB expressions generated by the worksharing
2254/// code generation support, whereas in non combined situations we would
2255/// just emit 0 and the LastIteration expression
2256/// This function is necessary due to the difference of the LB and UB
2257/// types for the RT emission routines for 'for_static_init' and
2258/// 'for_dispatch_init'
2259static std::pair<llvm::Value *, llvm::Value *>
2260emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
2261 const OMPExecutableDirective &S,
2262 Address LB, Address UB) {
2263 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2264 const Expr *IVExpr = LS.getIterationVariable();
2265 // when implementing a dynamic schedule for a 'for' combined with a
2266 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2267 // is not normalized as each team only executes its own assigned
2268 // distribute chunk
2269 QualType IteratorTy = IVExpr->getType();
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00002270 llvm::Value *LBVal =
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002271 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00002272 llvm::Value *UBVal =
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002273 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002274 return {LBVal, UBVal};
Hans Wennborged129ae2017-04-27 17:02:25 +00002275}
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002276
2277static void emitDistributeParallelForDistributeInnerBoundParams(
2278 CodeGenFunction &CGF, const OMPExecutableDirective &S,
2279 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2280 const auto &Dir = cast<OMPLoopDirective>(S);
2281 LValue LB =
2282 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
Akira Hatanakaf139ae32019-12-03 15:17:01 -08002283 llvm::Value *LBCast =
2284 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)),
2285 CGF.SizeTy, /*isSigned=*/false);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002286 CapturedVars.push_back(LBCast);
2287 LValue UB =
2288 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2289
Akira Hatanakaf139ae32019-12-03 15:17:01 -08002290 llvm::Value *UBCast =
2291 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)),
2292 CGF.SizeTy, /*isSigned=*/false);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002293 CapturedVars.push_back(UBCast);
Hans Wennborged129ae2017-04-27 17:02:25 +00002294}
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002295
2296static void
2297emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
2298 const OMPLoopDirective &S,
2299 CodeGenFunction::JumpDest LoopExit) {
2300 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00002301 PrePostActionTy &Action) {
2302 Action.Enter(CGF);
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00002303 bool HasCancel = false;
2304 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2305 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2306 HasCancel = D->hasCancel();
2307 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2308 HasCancel = D->hasCancel();
Alexey Bataev16e79882017-11-22 21:12:03 +00002309 else if (const auto *D =
2310 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2311 HasCancel = D->hasCancel();
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00002312 }
2313 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
2314 HasCancel);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002315 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
2316 emitDistributeParallelForInnerBounds,
2317 emitDistributeParallelForDispatchBounds);
2318 };
2319
2320 emitCommonOMPParallelDirective(
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00002321 CGF, S,
2322 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2323 CGInlinedWorksharingLoop,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002324 emitDistributeParallelForDistributeInnerBoundParams);
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002325}
2326
Carlo Bertolli9925f152016-06-27 14:55:37 +00002327void CodeGenFunction::EmitOMPDistributeParallelForDirective(
2328 const OMPDistributeParallelForDirective &S) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002329 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2330 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2331 S.getDistInc());
2332 };
Alexey Bataev475a7442018-01-12 19:39:11 +00002333 OMPLexicalScope Scope(*this, S, OMPD_parallel);
Alexey Bataev10a54312017-11-27 16:54:08 +00002334 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
Carlo Bertolli9925f152016-06-27 14:55:37 +00002335}
2336
Kelvin Li4a39add2016-07-05 05:00:15 +00002337void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
2338 const OMPDistributeParallelForSimdDirective &S) {
Alexey Bataev0b49f9e2017-11-27 19:38:58 +00002339 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2340 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2341 S.getDistInc());
2342 };
Alexey Bataev475a7442018-01-12 19:39:11 +00002343 OMPLexicalScope Scope(*this, S, OMPD_parallel);
Alexey Bataev0b49f9e2017-11-27 19:38:58 +00002344 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
Kelvin Li4a39add2016-07-05 05:00:15 +00002345}
Kelvin Li787f3fc2016-07-06 04:45:38 +00002346
2347void CodeGenFunction::EmitOMPDistributeSimdDirective(
2348 const OMPDistributeSimdDirective &S) {
Alexey Bataev617db5f2017-12-04 15:38:33 +00002349 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2350 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
2351 };
Alexey Bataev475a7442018-01-12 19:39:11 +00002352 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev617db5f2017-12-04 15:38:33 +00002353 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
Kelvin Li787f3fc2016-07-06 04:45:38 +00002354}
2355
Alexey Bataevf8365372017-11-17 17:57:25 +00002356void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
2357 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2358 // Emit SPMD target parallel for region as a standalone region.
2359 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2360 emitOMPSimdRegion(CGF, S, Action);
2361 };
2362 llvm::Function *Fn;
2363 llvm::Constant *Addr;
2364 // Emit target region as a standalone region.
2365 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2366 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2367 assert(Fn && Addr && "Target device function emission failed.");
2368}
2369
Kelvin Li986330c2016-07-20 22:57:10 +00002370void CodeGenFunction::EmitOMPTargetSimdDirective(
2371 const OMPTargetSimdDirective &S) {
Alexey Bataevf8365372017-11-17 17:57:25 +00002372 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2373 emitOMPSimdRegion(CGF, S, Action);
2374 };
2375 emitCommonOMPTargetDirective(*this, S, CodeGen);
Kelvin Li986330c2016-07-20 22:57:10 +00002376}
2377
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002378namespace {
2379 struct ScheduleKindModifiersTy {
2380 OpenMPScheduleClauseKind Kind;
2381 OpenMPScheduleClauseModifier M1;
2382 OpenMPScheduleClauseModifier M2;
2383 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2384 OpenMPScheduleClauseModifier M1,
2385 OpenMPScheduleClauseModifier M2)
2386 : Kind(Kind), M1(M1), M2(M2) {}
2387 };
2388} // namespace
2389
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002390bool CodeGenFunction::EmitOMPWorksharingLoop(
2391 const OMPLoopDirective &S, Expr *EUB,
2392 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2393 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
Alexander Musmanc6388682014-12-15 07:07:06 +00002394 // Emit the loop iteration variable.
Alexey Bataevddf3db92018-04-13 17:31:06 +00002395 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2396 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
Alexander Musmanc6388682014-12-15 07:07:06 +00002397 EmitVarDecl(*IVDecl);
2398
2399 // Emit the iterations count variable.
2400 // If it is not a variable, Sema decided to calculate iterations count on each
2401 // iteration (e.g., it is foldable into a constant).
Alexey Bataevddf3db92018-04-13 17:31:06 +00002402 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
Alexander Musmanc6388682014-12-15 07:07:06 +00002403 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2404 // Emit calculation of the iterations count.
2405 EmitIgnoredExpr(S.getCalcLastIteration());
2406 }
2407
Alexey Bataevddf3db92018-04-13 17:31:06 +00002408 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
Alexander Musmanc6388682014-12-15 07:07:06 +00002409
Alexey Bataev38e89532015-04-16 04:54:05 +00002410 bool HasLastprivateClause;
Alexander Musmanc6388682014-12-15 07:07:06 +00002411 // Check pre-condition.
2412 {
Alexey Bataev5a3af132016-03-29 08:58:54 +00002413 OMPLoopScope PreInitScope(*this, S);
Alexander Musmanc6388682014-12-15 07:07:06 +00002414 // Skip the entire loop if we don't meet the precondition.
Alexey Bataev62dbb972015-04-22 11:59:37 +00002415 // If the condition constant folds and can be elided, avoid emitting the
2416 // whole loop.
2417 bool CondConstant;
2418 llvm::BasicBlock *ContBlock = nullptr;
2419 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2420 if (!CondConstant)
2421 return false;
2422 } else {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002423 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
Alexey Bataev62dbb972015-04-22 11:59:37 +00002424 ContBlock = createBasicBlock("omp.precond.end");
2425 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
Justin Bogner66242d62015-04-23 23:06:47 +00002426 getProfileCount(&S));
Alexey Bataev62dbb972015-04-22 11:59:37 +00002427 EmitBlock(ThenBlock);
Justin Bogner66242d62015-04-23 23:06:47 +00002428 incrementProfileCounter(&S);
Alexey Bataev62dbb972015-04-22 11:59:37 +00002429 }
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00002430
Alexey Bataevea33dee2018-02-15 23:39:43 +00002431 RunCleanupsScope DoacrossCleanupScope(*this);
Alexey Bataev8b427062016-05-25 12:36:08 +00002432 bool Ordered = false;
Alexey Bataevddf3db92018-04-13 17:31:06 +00002433 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
Alexey Bataev8b427062016-05-25 12:36:08 +00002434 if (OrderedClause->getNumForLoops())
Alexey Bataevf138fda2018-08-13 19:04:24 +00002435 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
Alexey Bataev8b427062016-05-25 12:36:08 +00002436 else
2437 Ordered = true;
2438 }
2439
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002440 llvm::DenseSet<const Expr *> EmittedFinals;
Alexey Bataev58e5bdb2015-06-18 04:45:29 +00002441 emitAlignedClause(*this, S);
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00002442 bool HasLinears = EmitOMPLinearClauseInit(S);
Alexey Bataevef549a82016-03-09 09:49:09 +00002443 // Emit helper vars inits.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002444
2445 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2446 LValue LB = Bounds.first;
2447 LValue UB = Bounds.second;
Alexey Bataevef549a82016-03-09 09:49:09 +00002448 LValue ST =
2449 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2450 LValue IL =
2451 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2452
Alexander Musmanc6388682014-12-15 07:07:06 +00002453 // Emit 'then' code.
2454 {
Alexander Musmanc6388682014-12-15 07:07:06 +00002455 OMPPrivateScope LoopScope(*this);
Alexey Bataev8c3edfe2017-08-16 15:58:46 +00002456 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
Alexey Bataev69c62a92015-04-15 04:52:20 +00002457 // Emit implicit barrier to synchronize threads and avoid data races on
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00002458 // initialization of firstprivate variables and post-update of
2459 // lastprivate variables.
Alexey Bataev25e5b442015-09-15 12:52:43 +00002460 CGM.getOpenMPRuntime().emitBarrierCall(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002461 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
Alexey Bataev25e5b442015-09-15 12:52:43 +00002462 /*ForceSimpleCall=*/true);
Alexey Bataev69c62a92015-04-15 04:52:20 +00002463 }
Alexey Bataev50a64582015-04-22 12:24:45 +00002464 EmitOMPPrivateClause(S, LoopScope);
Alexey Bataev38e89532015-04-16 04:54:05 +00002465 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
Alexey Bataev7ebe5fd2015-04-22 13:43:03 +00002466 EmitOMPReductionClauseInit(S, LoopScope);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002467 EmitOMPPrivateLoopCounters(S, LoopScope);
2468 EmitOMPLinearClause(S, LoopScope);
Alexander Musman7931b982015-03-16 07:14:41 +00002469 (void)LoopScope.Privatize();
Alexey Bataev60705422018-10-30 15:50:12 +00002470 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2471 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
Alexander Musmanc6388682014-12-15 07:07:06 +00002472
2473 // Detect the loop schedule kind and chunk.
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00002474 const Expr *ChunkExpr = nullptr;
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002475 OpenMPScheduleTy ScheduleKind;
Alexey Bataevddf3db92018-04-13 17:31:06 +00002476 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002477 ScheduleKind.Schedule = C->getScheduleKind();
2478 ScheduleKind.M1 = C->getFirstScheduleModifier();
2479 ScheduleKind.M2 = C->getSecondScheduleModifier();
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00002480 ChunkExpr = C->getChunkSize();
Gheorghe-Teodor Bercea8233af92018-09-27 20:29:00 +00002481 } else {
2482 // Default behaviour for schedule clause.
2483 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00002484 *this, S, ScheduleKind.Schedule, ChunkExpr);
2485 }
2486 bool HasChunkSizeOne = false;
2487 llvm::Value *Chunk = nullptr;
2488 if (ChunkExpr) {
2489 Chunk = EmitScalarExpr(ChunkExpr);
2490 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2491 S.getIterationVariable()->getType(),
2492 S.getBeginLoc());
Fangrui Song407659a2018-11-30 23:41:18 +00002493 Expr::EvalResult Result;
2494 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2495 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00002496 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
Fangrui Song407659a2018-11-30 23:41:18 +00002497 }
Alexey Bataev3392d762016-02-16 11:18:12 +00002498 }
Alexander Musmanc6388682014-12-15 07:07:06 +00002499 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2500 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002501 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2502 // If the static schedule kind is specified or if the ordered clause is
2503 // specified, and if no monotonic modifier is specified, the effect will
2504 // be as if the monotonic modifier was specified.
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00002505 bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2506 /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2507 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
2508 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2509 /* Chunked */ Chunk != nullptr) ||
2510 StaticChunkedOne) &&
Alexey Bataevd7589ffe2015-05-20 13:12:48 +00002511 !Ordered) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002512 JumpDest LoopExit =
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002513 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
Alexey Bataev103f3c9e2019-11-20 15:59:03 -05002514 emitCommonSimdLoop(
2515 *this, S,
2516 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2517 if (isOpenMPSimdDirective(S.getDirectiveKind()))
2518 CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true);
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00002519 },
Alexey Bataev103f3c9e2019-11-20 15:59:03 -05002520 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
2521 &S, ScheduleKind, LoopExit,
2522 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2523 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2524 // When no chunk_size is specified, the iteration space is divided
2525 // into chunks that are approximately equal in size, and at most
2526 // one chunk is distributed to each thread. Note that the size of
2527 // the chunks is unspecified in this case.
2528 CGOpenMPRuntime::StaticRTInput StaticInit(
Akira Hatanakaf139ae32019-12-03 15:17:01 -08002529 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
2530 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
Alexey Bataev103f3c9e2019-11-20 15:59:03 -05002531 StaticChunkedOne ? Chunk : nullptr);
2532 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2533 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
2534 StaticInit);
2535 // UB = min(UB, GlobalUB);
2536 if (!StaticChunkedOne)
2537 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
2538 // IV = LB;
2539 CGF.EmitIgnoredExpr(S.getInit());
2540 // For unchunked static schedule generate:
2541 //
2542 // while (idx <= UB) {
2543 // BODY;
2544 // ++idx;
2545 // }
2546 //
2547 // For static schedule with chunk one:
2548 //
2549 // while (IV <= PrevUB) {
2550 // BODY;
2551 // IV += ST;
2552 // }
2553 CGF.EmitOMPInnerLoop(
2554 S, LoopScope.requiresCleanups(),
2555 StaticChunkedOne ? S.getCombinedParForInDistCond()
2556 : S.getCond(),
2557 StaticChunkedOne ? S.getDistInc() : S.getInc(),
2558 [&S, LoopExit](CodeGenFunction &CGF) {
2559 CGF.EmitOMPLoopBody(S, LoopExit);
2560 CGF.EmitStopPoint(&S);
2561 },
2562 [](CodeGenFunction &) {});
2563 });
Alexey Bataev0f34da12015-07-02 04:17:07 +00002564 EmitBlock(LoopExit.getBlock());
Alexander Musmanc6388682014-12-15 07:07:06 +00002565 // Tell the runtime we are done.
Alexey Bataev957d8562016-11-17 15:12:05 +00002566 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
Stephen Kelly1c301dc2018-08-09 21:09:38 +00002567 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
Alexey Bataevf43f7142017-09-06 16:17:35 +00002568 S.getDirectiveKind());
Alexey Bataev957d8562016-11-17 15:12:05 +00002569 };
2570 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002571 } else {
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002572 const bool IsMonotonic =
2573 Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2574 ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2575 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2576 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002577 // Emit the outer loop, which requests its work chunk [LB..UB] from
2578 // runtime and runs the inner loop to process it.
Akira Hatanakaf139ae32019-12-03 15:17:01 -08002579 const OMPLoopArguments LoopArguments(
2580 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
2581 IL.getAddress(*this), Chunk, EUB);
Alexey Bataeva6f2a142015-12-31 06:52:34 +00002582 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002583 LoopArguments, CGDispatchBounds);
Alexander Musmandf7a8e22015-01-22 08:49:35 +00002584 }
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002585 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002586 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2587 return CGF.Builder.CreateIsNotNull(
2588 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2589 });
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002590 }
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00002591 EmitOMPReductionClauseFinal(
2592 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2593 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2594 : /*Parallel only*/ OMPD_parallel);
Alexey Bataev61205072016-03-02 04:57:40 +00002595 // Emit post-update of the reduction variables if IsLastIter != 0.
2596 emitPostUpdateForReductionClause(
Alexey Bataevddf3db92018-04-13 17:31:06 +00002597 *this, S, [IL, &S](CodeGenFunction &CGF) {
Alexey Bataev61205072016-03-02 04:57:40 +00002598 return CGF.Builder.CreateIsNotNull(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002599 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
Alexey Bataev61205072016-03-02 04:57:40 +00002600 });
Alexey Bataev38e89532015-04-16 04:54:05 +00002601 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2602 if (HasLastprivateClause)
2603 EmitOMPLastprivateClauseFinal(
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002604 S, isOpenMPSimdDirective(S.getDirectiveKind()),
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002605 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
Alexander Musmanc6388682014-12-15 07:07:06 +00002606 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00002607 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
Alexey Bataevef549a82016-03-09 09:49:09 +00002608 return CGF.Builder.CreateIsNotNull(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002609 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
Alexey Bataevef549a82016-03-09 09:49:09 +00002610 });
Alexey Bataevea33dee2018-02-15 23:39:43 +00002611 DoacrossCleanupScope.ForceCleanup();
Alexander Musmanc6388682014-12-15 07:07:06 +00002612 // We're now done with the loop, so jump to the continuation block.
Alexey Bataev62dbb972015-04-22 11:59:37 +00002613 if (ContBlock) {
2614 EmitBranch(ContBlock);
Alexey Bataevddf3db92018-04-13 17:31:06 +00002615 EmitBlock(ContBlock, /*IsFinished=*/true);
Alexey Bataev62dbb972015-04-22 11:59:37 +00002616 }
Alexander Musmanc6388682014-12-15 07:07:06 +00002617 }
Alexey Bataev38e89532015-04-16 04:54:05 +00002618 return HasLastprivateClause;
Alexander Musmanc6388682014-12-15 07:07:06 +00002619}
2620
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002621/// The following two functions generate expressions for the loop lower
2622/// and upper bounds in case of static and dynamic (dispatch) schedule
2623/// of the associated 'for' or 'distribute' loop.
2624static std::pair<LValue, LValue>
2625emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002626 const auto &LS = cast<OMPLoopDirective>(S);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002627 LValue LB =
2628 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2629 LValue UB =
2630 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2631 return {LB, UB};
2632}
2633
2634/// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2635/// consider the lower and upper bound expressions generated by the
2636/// worksharing loop support, but we use 0 and the iteration space size as
2637/// constants
2638static std::pair<llvm::Value *, llvm::Value *>
2639emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2640 Address LB, Address UB) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002641 const auto &LS = cast<OMPLoopDirective>(S);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002642 const Expr *IVExpr = LS.getIterationVariable();
2643 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2644 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2645 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2646 return {LBVal, UBVal};
2647}
2648
Alexander Musmanc6388682014-12-15 07:07:06 +00002649void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
Alexey Bataev38e89532015-04-16 04:54:05 +00002650 bool HasLastprivates = false;
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002651 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2652 PrePostActionTy &) {
Alexey Bataev957d8562016-11-17 15:12:05 +00002653 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002654 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2655 emitForLoopBounds,
2656 emitDispatchForLoopBounds);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002657 };
Alexey Bataev3392d762016-02-16 11:18:12 +00002658 {
Alexey Bataev475a7442018-01-12 19:39:11 +00002659 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev3392d762016-02-16 11:18:12 +00002660 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2661 S.hasCancel());
2662 }
Alexander Musmanc6388682014-12-15 07:07:06 +00002663
2664 // Emit an implicit barrier at the end.
Alexey Bataevddf3db92018-04-13 17:31:06 +00002665 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002666 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
Alexey Bataevf29276e2014-06-18 04:14:57 +00002667}
Alexey Bataevd3f8dd22014-06-25 11:44:49 +00002668
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00002669void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00002670 bool HasLastprivates = false;
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002671 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2672 PrePostActionTy &) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002673 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2674 emitForLoopBounds,
2675 emitDispatchForLoopBounds);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002676 };
Alexey Bataev3392d762016-02-16 11:18:12 +00002677 {
Alexey Bataev475a7442018-01-12 19:39:11 +00002678 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev3392d762016-02-16 11:18:12 +00002679 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2680 }
Alexey Bataevcbdcbb72015-06-17 07:45:51 +00002681
2682 // Emit an implicit barrier at the end.
Alexey Bataevddf3db92018-04-13 17:31:06 +00002683 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002684 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
Alexander Musmanf82886e2014-09-18 05:12:34 +00002685}
2686
Alexey Bataev2df54a02015-03-12 08:53:29 +00002687static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2688 const Twine &Name,
2689 llvm::Value *Init = nullptr) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002690 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
Alexey Bataev2df54a02015-03-12 08:53:29 +00002691 if (Init)
Akira Hatanaka642f7992016-10-18 19:05:41 +00002692 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
Alexey Bataev2df54a02015-03-12 08:53:29 +00002693 return LVal;
Alexey Bataevd3f8dd22014-06-25 11:44:49 +00002694}
2695
Alexey Bataev3392d762016-02-16 11:18:12 +00002696void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002697 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
2698 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002699 bool HasLastprivates = false;
Alexey Bataevddf3db92018-04-13 17:31:06 +00002700 auto &&CodeGen = [&S, CapturedStmt, CS,
2701 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2702 ASTContext &C = CGF.getContext();
2703 QualType KmpInt32Ty =
2704 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002705 // Emit helper vars inits.
2706 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2707 CGF.Builder.getInt32(0));
Alexey Bataevddf3db92018-04-13 17:31:06 +00002708 llvm::ConstantInt *GlobalUBVal = CS != nullptr
2709 ? CGF.Builder.getInt32(CS->size() - 1)
2710 : CGF.Builder.getInt32(0);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002711 LValue UB =
2712 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2713 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2714 CGF.Builder.getInt32(1));
2715 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2716 CGF.Builder.getInt32(0));
2717 // Loop counter.
2718 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002719 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002720 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002721 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002722 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2723 // Generate condition for loop.
2724 BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002725 OK_Ordinary, S.getBeginLoc(), FPOptions());
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002726 // Increment for loop counter.
2727 UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002728 S.getBeginLoc(), true);
Alexey Bataevddf3db92018-04-13 17:31:06 +00002729 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002730 // Iterate through all sections and emit a switch construct:
2731 // switch (IV) {
2732 // case 0:
2733 // <SectionStmt[0]>;
2734 // break;
2735 // ...
2736 // case <NumSection> - 1:
2737 // <SectionStmt[<NumSection> - 1]>;
2738 // break;
2739 // }
2740 // .omp.sections.exit:
Alexey Bataevddf3db92018-04-13 17:31:06 +00002741 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2742 llvm::SwitchInst *SwitchStmt =
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002743 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00002744 ExitBB, CS == nullptr ? 1 : CS->size());
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002745 if (CS) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002746 unsigned CaseNumber = 0;
Alexey Bataevddf3db92018-04-13 17:31:06 +00002747 for (const Stmt *SubStmt : CS->children()) {
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002748 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2749 CGF.EmitBlock(CaseBB);
2750 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
Benjamin Kramer642f1732015-07-02 21:03:14 +00002751 CGF.EmitStmt(SubStmt);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002752 CGF.EmitBranch(ExitBB);
Benjamin Kramer642f1732015-07-02 21:03:14 +00002753 ++CaseNumber;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002754 }
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002755 } else {
Alexey Bataevddf3db92018-04-13 17:31:06 +00002756 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002757 CGF.EmitBlock(CaseBB);
2758 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
Alexey Bataevddf3db92018-04-13 17:31:06 +00002759 CGF.EmitStmt(CapturedStmt);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002760 CGF.EmitBranch(ExitBB);
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002761 }
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002762 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
Alexey Bataev2df54a02015-03-12 08:53:29 +00002763 };
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002764
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002765 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2766 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
Alexey Bataev9efc03b2015-04-27 04:34:03 +00002767 // Emit implicit barrier to synchronize threads and avoid data races on
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00002768 // initialization of firstprivate variables and post-update of lastprivate
2769 // variables.
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002770 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002771 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002772 /*ForceSimpleCall=*/true);
Alexey Bataev9efc03b2015-04-27 04:34:03 +00002773 }
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002774 CGF.EmitOMPPrivateClause(S, LoopScope);
2775 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2776 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2777 (void)LoopScope.Privatize();
Alexey Bataev60705422018-10-30 15:50:12 +00002778 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2779 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002780
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002781 // Emit static non-chunked loop.
Alexey Bataev9ebd7422016-05-10 09:57:36 +00002782 OpenMPScheduleTy ScheduleKind;
2783 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00002784 CGOpenMPRuntime::StaticRTInput StaticInit(
Akira Hatanakaf139ae32019-12-03 15:17:01 -08002785 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF),
2786 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF));
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002787 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002788 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002789 // UB = min(UB, GlobalUB);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002790 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
Alexey Bataevddf3db92018-04-13 17:31:06 +00002791 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002792 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2793 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2794 // IV = LB;
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002795 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002796 // while (idx <= UB) { BODY; ++idx; }
2797 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2798 [](CodeGenFunction &) {});
2799 // Tell the runtime we are done.
Alexey Bataev957d8562016-11-17 15:12:05 +00002800 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
Stephen Kelly1c301dc2018-08-09 21:09:38 +00002801 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
Alexey Bataevf43f7142017-09-06 16:17:35 +00002802 S.getDirectiveKind());
Alexey Bataev957d8562016-11-17 15:12:05 +00002803 };
2804 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00002805 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
Alexey Bataev61205072016-03-02 04:57:40 +00002806 // Emit post-update of the reduction variables if IsLastIter != 0.
Alexey Bataevddf3db92018-04-13 17:31:06 +00002807 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2808 return CGF.Builder.CreateIsNotNull(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002809 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
Alexey Bataevddf3db92018-04-13 17:31:06 +00002810 });
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002811
2812 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2813 if (HasLastprivates)
2814 CGF.EmitOMPLastprivateClauseFinal(
Alexey Bataev5dff95c2016-04-22 03:56:56 +00002815 S, /*NoFinals=*/false,
2816 CGF.Builder.CreateIsNotNull(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002817 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002818 };
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002819
2820 bool HasCancel = false;
2821 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2822 HasCancel = OSD->hasCancel();
2823 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2824 HasCancel = OPSD->hasCancel();
Alexey Bataev957d8562016-11-17 15:12:05 +00002825 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002826 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2827 HasCancel);
2828 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2829 // clause. Otherwise the barrier will be generated by the codegen for the
2830 // directive.
2831 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002832 // Emit implicit barrier to synchronize threads and avoid data races on
2833 // initialization of firstprivate variables.
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002834 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
Alexey Bataev3015bcc2016-01-22 08:56:50 +00002835 OMPD_unknown);
Alexey Bataev2cb9b952015-04-24 03:37:03 +00002836 }
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002837}
Alexey Bataev2df54a02015-03-12 08:53:29 +00002838
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002839void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
Alexey Bataev3392d762016-02-16 11:18:12 +00002840 {
Alexey Bataev475a7442018-01-12 19:39:11 +00002841 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev3392d762016-02-16 11:18:12 +00002842 EmitSections(S);
2843 }
Alexey Bataev2df54a02015-03-12 08:53:29 +00002844 // Emit an implicit barrier at the end.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002845 if (!S.getSingleClause<OMPNowaitClause>()) {
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002846 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
Alexey Bataev3392d762016-02-16 11:18:12 +00002847 OMPD_sections);
Alexey Bataevf2685682015-03-30 04:30:22 +00002848 }
Alexey Bataev2df54a02015-03-12 08:53:29 +00002849}
2850
2851void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002852 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev475a7442018-01-12 19:39:11 +00002853 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002854 };
Alexey Bataev475a7442018-01-12 19:39:11 +00002855 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev25e5b442015-09-15 12:52:43 +00002856 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2857 S.hasCancel());
Alexey Bataev1e0498a2014-06-26 08:21:58 +00002858}
2859
Alexey Bataev6956e2e2015-02-05 06:35:41 +00002860void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
Alexey Bataeva63048e2015-03-23 06:18:07 +00002861 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
Alexey Bataev420d45b2015-04-14 05:11:24 +00002862 llvm::SmallVector<const Expr *, 8> DestExprs;
Alexey Bataeva63048e2015-03-23 06:18:07 +00002863 llvm::SmallVector<const Expr *, 8> SrcExprs;
Alexey Bataeva63048e2015-03-23 06:18:07 +00002864 llvm::SmallVector<const Expr *, 8> AssignmentOps;
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002865 // Check if there are any 'copyprivate' clauses associated with this
Alexey Bataevcd8b6a22016-02-15 08:07:17 +00002866 // 'single' construct.
Alexey Bataeva63048e2015-03-23 06:18:07 +00002867 // Build a list of copyprivate variables along with helper expressions
2868 // (<source>, <destination>, <destination>=<source> expressions)
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002869 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
Alexey Bataeva63048e2015-03-23 06:18:07 +00002870 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
Alexey Bataev420d45b2015-04-14 05:11:24 +00002871 DestExprs.append(C->destination_exprs().begin(),
2872 C->destination_exprs().end());
Alexey Bataeva63048e2015-03-23 06:18:07 +00002873 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
Alexey Bataeva63048e2015-03-23 06:18:07 +00002874 AssignmentOps.append(C->assignment_ops().begin(),
2875 C->assignment_ops().end());
2876 }
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002877 // Emit code for 'single' region along with 'copyprivate' clauses
2878 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2879 Action.Enter(CGF);
2880 OMPPrivateScope SingleScope(CGF);
2881 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2882 CGF.EmitOMPPrivateClause(S, SingleScope);
2883 (void)SingleScope.Privatize();
Alexey Bataev475a7442018-01-12 19:39:11 +00002884 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002885 };
Alexey Bataev3392d762016-02-16 11:18:12 +00002886 {
Alexey Bataev475a7442018-01-12 19:39:11 +00002887 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002888 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
Alexey Bataev3392d762016-02-16 11:18:12 +00002889 CopyprivateVars, DestExprs,
2890 SrcExprs, AssignmentOps);
2891 }
2892 // Emit an implicit barrier at the end (to avoid data race on firstprivate
2893 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
Alexey Bataev417089f2016-02-17 13:19:37 +00002894 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
Alexey Bataev5521d782015-04-24 04:21:15 +00002895 CGM.getOpenMPRuntime().emitBarrierCall(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002896 *this, S.getBeginLoc(),
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00002897 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
Alexey Bataevf2685682015-03-30 04:30:22 +00002898 }
Alexey Bataevd1e40fb2014-06-26 12:05:45 +00002899}
2900
cchen47d60942019-12-05 13:43:48 -05002901static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002902 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2903 Action.Enter(CGF);
Alexey Bataev475a7442018-01-12 19:39:11 +00002904 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002905 };
cchen47d60942019-12-05 13:43:48 -05002906 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
2907}
2908
2909void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
Alexey Bataev475a7442018-01-12 19:39:11 +00002910 OMPLexicalScope Scope(*this, S, OMPD_unknown);
cchen47d60942019-12-05 13:43:48 -05002911 emitMaster(*this, S);
Alexander Musman80c22892014-07-17 08:54:58 +00002912}
2913
Alexey Bataev3a3bf0b2014-09-22 10:01:53 +00002914void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002915 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2916 Action.Enter(CGF);
Alexey Bataev475a7442018-01-12 19:39:11 +00002917 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00002918 };
Alexey Bataevddf3db92018-04-13 17:31:06 +00002919 const Expr *Hint = nullptr;
2920 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
Alexey Bataevfc57d162015-12-15 10:55:09 +00002921 Hint = HintClause->getHint();
Alexey Bataev475a7442018-01-12 19:39:11 +00002922 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataevfc57d162015-12-15 10:55:09 +00002923 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2924 S.getDirectiveName().getAsString(),
Stephen Kellyf2ceec42018-08-09 21:08:08 +00002925 CodeGen, S.getBeginLoc(), Hint);
Alexander Musmand9ed09f2014-07-21 09:42:05 +00002926}
2927
Alexey Bataev671605e2015-04-13 05:28:11 +00002928void CodeGenFunction::EmitOMPParallelForDirective(
2929 const OMPParallelForDirective &S) {
2930 // Emit directive as a combined directive that consists of two implicit
2931 // directives: 'parallel' with 'for' directive.
Alexey Bataevc99042b2018-03-15 18:10:54 +00002932 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2933 Action.Enter(CGF);
Alexey Bataev957d8562016-11-17 15:12:05 +00002934 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002935 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2936 emitDispatchForLoopBounds);
Alexey Bataev671605e2015-04-13 05:28:11 +00002937 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002938 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2939 emitEmptyBoundParameters);
Alexey Bataev4acb8592014-07-07 13:01:15 +00002940}
2941
Alexander Musmane4e893b2014-09-23 09:33:00 +00002942void CodeGenFunction::EmitOMPParallelForSimdDirective(
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00002943 const OMPParallelForSimdDirective &S) {
2944 // Emit directive as a combined directive that consists of two implicit
2945 // directives: 'parallel' with 'for' directive.
Alexey Bataevc99042b2018-03-15 18:10:54 +00002946 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2947 Action.Enter(CGF);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002948 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2949 emitDispatchForLoopBounds);
Alexey Bataev3b5b5c42015-06-18 10:10:12 +00002950 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002951 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2952 emitEmptyBoundParameters);
Alexander Musmane4e893b2014-09-23 09:33:00 +00002953}
2954
cchen47d60942019-12-05 13:43:48 -05002955void CodeGenFunction::EmitOMPParallelMasterDirective(
2956 const OMPParallelMasterDirective &S) {
2957 // Emit directive as a combined directive that consists of two implicit
2958 // directives: 'parallel' with 'master' directive.
2959 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2960 Action.Enter(CGF);
2961 OMPPrivateScope PrivateScope(CGF);
2962 bool Copyins = CGF.EmitOMPCopyinClause(S);
2963 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
2964 if (Copyins) {
2965 // Emit implicit barrier to synchronize threads and avoid data races on
2966 // propagation master's thread values of threadprivate variables to local
2967 // instances of that variables of all other implicit threads.
2968 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2969 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2970 /*ForceSimpleCall=*/true);
2971 }
2972 CGF.EmitOMPPrivateClause(S, PrivateScope);
2973 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
2974 (void)PrivateScope.Privatize();
2975 emitMaster(CGF, S);
2976 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2977 };
2978 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
2979 emitEmptyBoundParameters);
2980 emitPostUpdateForReductionClause(*this, S,
2981 [](CodeGenFunction &) { return nullptr; });
2982}
2983
Alexey Bataev84d0b3e2014-07-08 08:12:03 +00002984void CodeGenFunction::EmitOMPParallelSectionsDirective(
Alexey Bataev68adb7d2015-04-14 03:29:22 +00002985 const OMPParallelSectionsDirective &S) {
2986 // Emit directive as a combined directive that consists of two implicit
2987 // directives: 'parallel' with 'sections' directive.
Alexey Bataevc99042b2018-03-15 18:10:54 +00002988 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2989 Action.Enter(CGF);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00002990 CGF.EmitSections(S);
2991 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00002992 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2993 emitEmptyBoundParameters);
Alexey Bataev84d0b3e2014-07-08 08:12:03 +00002994}
2995
Alexey Bataev475a7442018-01-12 19:39:11 +00002996void CodeGenFunction::EmitOMPTaskBasedDirective(
2997 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2998 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2999 OMPTaskDataTy &Data) {
Alexey Bataev62b63b12015-03-10 07:28:44 +00003000 // Emit outlined function for task construct.
Alexey Bataev475a7442018-01-12 19:39:11 +00003001 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003002 auto I = CS->getCapturedDecl()->param_begin();
3003 auto PartId = std::next(I);
3004 auto TaskT = std::next(I, 4);
Alexey Bataev24b5bae2016-04-28 09:23:51 +00003005 // Check if the task is final
3006 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
3007 // If the condition constant folds and can be elided, try to avoid emitting
3008 // the condition and the dead arm of the if/else.
Alexey Bataevddf3db92018-04-13 17:31:06 +00003009 const Expr *Cond = Clause->getCondition();
Alexey Bataev24b5bae2016-04-28 09:23:51 +00003010 bool CondConstant;
3011 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
3012 Data.Final.setInt(CondConstant);
3013 else
3014 Data.Final.setPointer(EvaluateExprAsBool(Cond));
3015 } else {
3016 // By default the task is not final.
3017 Data.Final.setInt(/*IntVal=*/false);
3018 }
Alexey Bataev1e1e2862016-05-10 12:21:02 +00003019 // Check if the task has 'priority' clause.
3020 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00003021 const Expr *Prio = Clause->getPriority();
Alexey Bataev5140e742016-07-19 04:21:09 +00003022 Data.Priority.setInt(/*IntVal=*/true);
Alexey Bataevad537bb2016-05-30 09:06:50 +00003023 Data.Priority.setPointer(EmitScalarConversion(
3024 EmitScalarExpr(Prio), Prio->getType(),
3025 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
3026 Prio->getExprLoc()));
Alexey Bataev1e1e2862016-05-10 12:21:02 +00003027 }
Alexey Bataev62b63b12015-03-10 07:28:44 +00003028 // The first function argument for tasks is a thread id, the second one is a
3029 // part id (0 for tied tasks, >=0 for untied task).
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003030 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
3031 // Get list of private variables.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00003032 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003033 auto IRef = C->varlist_begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00003034 for (const Expr *IInit : C->private_copies()) {
3035 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003036 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataev7292c292016-04-25 12:22:29 +00003037 Data.PrivateVars.push_back(*IRef);
3038 Data.PrivateCopies.push_back(IInit);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003039 }
3040 ++IRef;
3041 }
3042 }
3043 EmittedAsPrivate.clear();
3044 // Get list of firstprivate variables.
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00003045 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003046 auto IRef = C->varlist_begin();
3047 auto IElemInitRef = C->inits().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00003048 for (const Expr *IInit : C->private_copies()) {
3049 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003050 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
Alexey Bataev7292c292016-04-25 12:22:29 +00003051 Data.FirstprivateVars.push_back(*IRef);
3052 Data.FirstprivateCopies.push_back(IInit);
3053 Data.FirstprivateInits.push_back(*IElemInitRef);
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003054 }
Richard Trieucc3949d2016-02-18 22:34:54 +00003055 ++IRef;
3056 ++IElemInitRef;
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003057 }
3058 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00003059 // Get list of lastprivate variables (for taskloops).
3060 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
3061 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
3062 auto IRef = C->varlist_begin();
3063 auto ID = C->destination_exprs().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00003064 for (const Expr *IInit : C->private_copies()) {
3065 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
Alexey Bataevf93095a2016-05-05 08:46:22 +00003066 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
3067 Data.LastprivateVars.push_back(*IRef);
3068 Data.LastprivateCopies.push_back(IInit);
3069 }
3070 LastprivateDstsOrigs.insert(
3071 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
3072 cast<DeclRefExpr>(*IRef)});
3073 ++IRef;
3074 ++ID;
3075 }
3076 }
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003077 SmallVector<const Expr *, 4> LHSs;
3078 SmallVector<const Expr *, 4> RHSs;
3079 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3080 auto IPriv = C->privates().begin();
3081 auto IRed = C->reduction_ops().begin();
3082 auto ILHS = C->lhs_exprs().begin();
3083 auto IRHS = C->rhs_exprs().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00003084 for (const Expr *Ref : C->varlists()) {
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003085 Data.ReductionVars.emplace_back(Ref);
3086 Data.ReductionCopies.emplace_back(*IPriv);
3087 Data.ReductionOps.emplace_back(*IRed);
3088 LHSs.emplace_back(*ILHS);
3089 RHSs.emplace_back(*IRHS);
3090 std::advance(IPriv, 1);
3091 std::advance(IRed, 1);
3092 std::advance(ILHS, 1);
3093 std::advance(IRHS, 1);
3094 }
3095 }
3096 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003097 *this, S.getBeginLoc(), LHSs, RHSs, Data);
Alexey Bataev1d2353d2015-06-24 11:01:36 +00003098 // Build list of dependences.
Alexey Bataev7292c292016-04-25 12:22:29 +00003099 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
Alexey Bataevddf3db92018-04-13 17:31:06 +00003100 for (const Expr *IRef : C->varlists())
Alexey Bataev43a919f2018-04-13 17:48:43 +00003101 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
Alexey Bataev475a7442018-01-12 19:39:11 +00003102 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
3103 CapturedRegion](CodeGenFunction &CGF,
3104 PrePostActionTy &Action) {
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003105 // Set proper addresses for generated private copies.
Alexey Bataev7292c292016-04-25 12:22:29 +00003106 OMPPrivateScope Scope(CGF);
Alexey Bataevf93095a2016-05-05 08:46:22 +00003107 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
3108 !Data.LastprivateVars.empty()) {
James Y Knight9871db02019-02-05 16:42:33 +00003109 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3110 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
Alexey Bataev3c595a62017-08-14 15:01:03 +00003111 enum { PrivatesParam = 2, CopyFnParam = 3 };
Alexey Bataevddf3db92018-04-13 17:31:06 +00003112 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3113 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3114 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3115 CS->getCapturedDecl()->getParam(PrivatesParam)));
Alexey Bataev48591dd2016-04-20 04:01:36 +00003116 // Map privates.
3117 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3118 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3119 CallArgs.push_back(PrivatesPtr);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003120 for (const Expr *E : Data.PrivateVars) {
3121 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
Alexey Bataev48591dd2016-04-20 04:01:36 +00003122 Address PrivatePtr = CGF.CreateMemTemp(
3123 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
Alexey Bataevddf3db92018-04-13 17:31:06 +00003124 PrivatePtrs.emplace_back(VD, PrivatePtr);
Alexey Bataev48591dd2016-04-20 04:01:36 +00003125 CallArgs.push_back(PrivatePtr.getPointer());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003126 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00003127 for (const Expr *E : Data.FirstprivateVars) {
3128 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
Alexey Bataev48591dd2016-04-20 04:01:36 +00003129 Address PrivatePtr =
3130 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3131 ".firstpriv.ptr.addr");
Alexey Bataevddf3db92018-04-13 17:31:06 +00003132 PrivatePtrs.emplace_back(VD, PrivatePtr);
Alexey Bataev48591dd2016-04-20 04:01:36 +00003133 CallArgs.push_back(PrivatePtr.getPointer());
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003134 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00003135 for (const Expr *E : Data.LastprivateVars) {
3136 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
Alexey Bataevf93095a2016-05-05 08:46:22 +00003137 Address PrivatePtr =
3138 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3139 ".lastpriv.ptr.addr");
Alexey Bataevddf3db92018-04-13 17:31:06 +00003140 PrivatePtrs.emplace_back(VD, PrivatePtr);
Alexey Bataevf93095a2016-05-05 08:46:22 +00003141 CallArgs.push_back(PrivatePtr.getPointer());
3142 }
James Y Knight9871db02019-02-05 16:42:33 +00003143 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3144 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003145 for (const auto &Pair : LastprivateDstsOrigs) {
3146 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
Bruno Ricci5fc4db72018-12-21 14:10:18 +00003147 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
3148 /*RefersToEnclosingVariableOrCapture=*/
3149 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
3150 Pair.second->getType(), VK_LValue,
3151 Pair.second->getExprLoc());
Alexey Bataevf93095a2016-05-05 08:46:22 +00003152 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003153 return CGF.EmitLValue(&DRE).getAddress(CGF);
Alexey Bataevf93095a2016-05-05 08:46:22 +00003154 });
3155 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00003156 for (const auto &Pair : PrivatePtrs) {
Alexey Bataev48591dd2016-04-20 04:01:36 +00003157 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3158 CGF.getContext().getDeclAlign(Pair.first));
3159 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3160 }
Alexey Bataev3ae88e22015-05-22 08:56:35 +00003161 }
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003162 if (Data.Reductions) {
Alexey Bataev475a7442018-01-12 19:39:11 +00003163 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003164 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
3165 Data.ReductionOps);
3166 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
3167 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
3168 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
3169 RedCG.emitSharedLValue(CGF, Cnt);
3170 RedCG.emitAggregateType(CGF, Cnt);
Alexey Bataev2e0cbe502018-03-08 15:24:08 +00003171 // FIXME: This must removed once the runtime library is fixed.
3172 // Emit required threadprivate variables for
Raphael Isemannb23ccec2018-12-10 12:37:46 +00003173 // initializer/combiner/finalizer.
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003174 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
Alexey Bataev2e0cbe502018-03-08 15:24:08 +00003175 RedCG, Cnt);
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003176 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003177 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003178 Replacement =
3179 Address(CGF.EmitScalarConversion(
3180 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3181 CGF.getContext().getPointerType(
3182 Data.ReductionCopies[Cnt]->getType()),
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003183 Data.ReductionCopies[Cnt]->getExprLoc()),
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003184 Replacement.getAlignment());
3185 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3186 Scope.addPrivate(RedCG.getBaseDecl(Cnt),
3187 [Replacement]() { return Replacement; });
Alexey Bataevbe5a8b42017-07-17 13:30:36 +00003188 }
3189 }
Alexey Bataev88202be2017-07-27 13:20:36 +00003190 // Privatize all private variables except for in_reduction items.
Alexey Bataev48591dd2016-04-20 04:01:36 +00003191 (void)Scope.Privatize();
Alexey Bataev88202be2017-07-27 13:20:36 +00003192 SmallVector<const Expr *, 4> InRedVars;
3193 SmallVector<const Expr *, 4> InRedPrivs;
3194 SmallVector<const Expr *, 4> InRedOps;
3195 SmallVector<const Expr *, 4> TaskgroupDescriptors;
3196 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
3197 auto IPriv = C->privates().begin();
3198 auto IRed = C->reduction_ops().begin();
3199 auto ITD = C->taskgroup_descriptors().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00003200 for (const Expr *Ref : C->varlists()) {
Alexey Bataev88202be2017-07-27 13:20:36 +00003201 InRedVars.emplace_back(Ref);
3202 InRedPrivs.emplace_back(*IPriv);
3203 InRedOps.emplace_back(*IRed);
3204 TaskgroupDescriptors.emplace_back(*ITD);
3205 std::advance(IPriv, 1);
3206 std::advance(IRed, 1);
3207 std::advance(ITD, 1);
3208 }
3209 }
3210 // Privatize in_reduction items here, because taskgroup descriptors must be
3211 // privatized earlier.
3212 OMPPrivateScope InRedScope(CGF);
3213 if (!InRedVars.empty()) {
3214 ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3215 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3216 RedCG.emitSharedLValue(CGF, Cnt);
3217 RedCG.emitAggregateType(CGF, Cnt);
3218 // The taskgroup descriptor variable is always implicit firstprivate and
Raphael Isemannb23ccec2018-12-10 12:37:46 +00003219 // privatized already during processing of the firstprivates.
Alexey Bataev2e0cbe502018-03-08 15:24:08 +00003220 // FIXME: This must removed once the runtime library is fixed.
3221 // Emit required threadprivate variables for
Raphael Isemannb23ccec2018-12-10 12:37:46 +00003222 // initializer/combiner/finalizer.
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003223 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
Alexey Bataev2e0cbe502018-03-08 15:24:08 +00003224 RedCG, Cnt);
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003225 llvm::Value *ReductionsPtr =
3226 CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3227 TaskgroupDescriptors[Cnt]->getExprLoc());
Alexey Bataev88202be2017-07-27 13:20:36 +00003228 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003229 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
Alexey Bataev88202be2017-07-27 13:20:36 +00003230 Replacement = Address(
3231 CGF.EmitScalarConversion(
3232 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3233 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003234 InRedPrivs[Cnt]->getExprLoc()),
Alexey Bataev88202be2017-07-27 13:20:36 +00003235 Replacement.getAlignment());
3236 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3237 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3238 [Replacement]() { return Replacement; });
Alexey Bataev88202be2017-07-27 13:20:36 +00003239 }
3240 }
3241 (void)InRedScope.Privatize();
Alexey Bataev48591dd2016-04-20 04:01:36 +00003242
3243 Action.Enter(CGF);
Alexey Bataev7292c292016-04-25 12:22:29 +00003244 BodyGen(CGF);
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00003245 };
James Y Knight9871db02019-02-05 16:42:33 +00003246 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
Alexey Bataev7292c292016-04-25 12:22:29 +00003247 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3248 Data.NumberOfParts);
Alexey Bataev5bbcead2019-10-14 17:17:41 +00003249 OMPLexicalScope Scope(*this, S, llvm::None,
Alexey Bataev61205822019-12-04 09:50:21 -05003250 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3251 !isOpenMPSimdDirective(S.getDirectiveKind()));
Alexey Bataev7292c292016-04-25 12:22:29 +00003252 TaskGen(*this, OutlinedFn, Data);
3253}
3254
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003255static ImplicitParamDecl *
3256createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003257 QualType Ty, CapturedDecl *CD,
3258 SourceLocation Loc) {
3259 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3260 ImplicitParamDecl::Other);
3261 auto *OrigRef = DeclRefExpr::Create(
3262 C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3263 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3264 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3265 ImplicitParamDecl::Other);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003266 auto *PrivateRef = DeclRefExpr::Create(
3267 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003268 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003269 QualType ElemType = C.getBaseElementType(Ty);
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003270 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3271 ImplicitParamDecl::Other);
3272 auto *InitRef = DeclRefExpr::Create(
3273 C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3274 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003275 PrivateVD->setInitStyle(VarDecl::CInit);
3276 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3277 InitRef, /*BasePath=*/nullptr,
3278 VK_RValue));
3279 Data.FirstprivateVars.emplace_back(OrigRef);
3280 Data.FirstprivateCopies.emplace_back(PrivateRef);
3281 Data.FirstprivateInits.emplace_back(InitRef);
3282 return OrigVD;
3283}
3284
3285void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
3286 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3287 OMPTargetDataInfo &InputInfo) {
3288 // Emit outlined function for task construct.
Alexey Bataevddf3db92018-04-13 17:31:06 +00003289 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3290 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3291 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3292 auto I = CS->getCapturedDecl()->param_begin();
3293 auto PartId = std::next(I);
3294 auto TaskT = std::next(I, 4);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003295 OMPTaskDataTy Data;
3296 // The task is not final.
3297 Data.Final.setInt(/*IntVal=*/false);
3298 // Get list of firstprivate variables.
3299 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3300 auto IRef = C->varlist_begin();
3301 auto IElemInitRef = C->inits().begin();
3302 for (auto *IInit : C->private_copies()) {
3303 Data.FirstprivateVars.push_back(*IRef);
3304 Data.FirstprivateCopies.push_back(IInit);
3305 Data.FirstprivateInits.push_back(*IElemInitRef);
3306 ++IRef;
3307 ++IElemInitRef;
3308 }
3309 }
3310 OMPPrivateScope TargetScope(*this);
3311 VarDecl *BPVD = nullptr;
3312 VarDecl *PVD = nullptr;
3313 VarDecl *SVD = nullptr;
3314 if (InputInfo.NumberOfTargetItems > 0) {
3315 auto *CD = CapturedDecl::Create(
3316 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3317 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3318 QualType BaseAndPointersType = getContext().getConstantArrayType(
Richard Smith772e2662019-10-04 01:25:59 +00003319 getContext().VoidPtrTy, ArrSize, nullptr, ArrayType::Normal,
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003320 /*IndexTypeQuals=*/0);
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003321 BPVD = createImplicitFirstprivateForType(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003322 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003323 PVD = createImplicitFirstprivateForType(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003324 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003325 QualType SizesType = getContext().getConstantArrayType(
Alexey Bataeva90fc662019-06-25 16:00:43 +00003326 getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
Richard Smith772e2662019-10-04 01:25:59 +00003327 ArrSize, nullptr, ArrayType::Normal,
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003328 /*IndexTypeQuals=*/0);
Alexey Bataeva9b9cc02018-01-23 18:12:38 +00003329 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003330 S.getBeginLoc());
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003331 TargetScope.addPrivate(
3332 BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3333 TargetScope.addPrivate(PVD,
3334 [&InputInfo]() { return InputInfo.PointersArray; });
3335 TargetScope.addPrivate(SVD,
3336 [&InputInfo]() { return InputInfo.SizesArray; });
3337 }
3338 (void)TargetScope.Privatize();
3339 // Build list of dependences.
3340 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
Alexey Bataevddf3db92018-04-13 17:31:06 +00003341 for (const Expr *IRef : C->varlists())
Alexey Bataev43a919f2018-04-13 17:48:43 +00003342 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003343 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3344 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3345 // Set proper addresses for generated private copies.
3346 OMPPrivateScope Scope(CGF);
3347 if (!Data.FirstprivateVars.empty()) {
James Y Knight9871db02019-02-05 16:42:33 +00003348 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3349 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003350 enum { PrivatesParam = 2, CopyFnParam = 3 };
Alexey Bataevddf3db92018-04-13 17:31:06 +00003351 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3352 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3353 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3354 CS->getCapturedDecl()->getParam(PrivatesParam)));
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003355 // Map privates.
3356 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3357 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3358 CallArgs.push_back(PrivatesPtr);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003359 for (const Expr *E : Data.FirstprivateVars) {
3360 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003361 Address PrivatePtr =
3362 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3363 ".firstpriv.ptr.addr");
Alexey Bataevddf3db92018-04-13 17:31:06 +00003364 PrivatePtrs.emplace_back(VD, PrivatePtr);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003365 CallArgs.push_back(PrivatePtr.getPointer());
3366 }
James Y Knight9871db02019-02-05 16:42:33 +00003367 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3368 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003369 for (const auto &Pair : PrivatePtrs) {
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003370 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3371 CGF.getContext().getDeclAlign(Pair.first));
3372 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3373 }
3374 }
3375 // Privatize all private variables except for in_reduction items.
3376 (void)Scope.Privatize();
Alexey Bataev8451efa2018-01-15 19:06:12 +00003377 if (InputInfo.NumberOfTargetItems > 0) {
3378 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
James Y Knight751fe282019-02-09 22:22:28 +00003379 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
Alexey Bataev8451efa2018-01-15 19:06:12 +00003380 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
James Y Knight751fe282019-02-09 22:22:28 +00003381 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
Alexey Bataev8451efa2018-01-15 19:06:12 +00003382 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
James Y Knight751fe282019-02-09 22:22:28 +00003383 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
Alexey Bataev8451efa2018-01-15 19:06:12 +00003384 }
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003385
3386 Action.Enter(CGF);
Alexey Bataev475a7442018-01-12 19:39:11 +00003387 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003388 BodyGen(CGF);
3389 };
James Y Knight9871db02019-02-05 16:42:33 +00003390 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003391 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3392 Data.NumberOfParts);
3393 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3394 IntegerLiteral IfCond(getContext(), TrueOrFalse,
3395 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3396 SourceLocation());
3397
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003398 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
Alexey Bataevd2202ca2017-12-27 17:58:32 +00003399 SharedsTy, CapturedStruct, &IfCond, Data);
3400}
3401
Alexey Bataev7292c292016-04-25 12:22:29 +00003402void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
3403 // Emit outlined function for task construct.
Alexey Bataev475a7442018-01-12 19:39:11 +00003404 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003405 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3406 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
Alexey Bataev1d677132015-04-22 13:57:31 +00003407 const Expr *IfCond = nullptr;
Alexey Bataev7371aa32015-09-03 08:45:56 +00003408 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3409 if (C->getNameModifier() == OMPD_unknown ||
3410 C->getNameModifier() == OMPD_task) {
3411 IfCond = C->getCondition();
3412 break;
3413 }
Alexey Bataev1d677132015-04-22 13:57:31 +00003414 }
Alexey Bataev7292c292016-04-25 12:22:29 +00003415
Alexey Bataev24b5bae2016-04-28 09:23:51 +00003416 OMPTaskDataTy Data;
3417 // Check if we should emit tied or untied task.
3418 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
Alexey Bataev7292c292016-04-25 12:22:29 +00003419 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3420 CGF.EmitStmt(CS->getCapturedStmt());
3421 };
Alexey Bataev24b5bae2016-04-28 09:23:51 +00003422 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
James Y Knight9871db02019-02-05 16:42:33 +00003423 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
Alexey Bataev24b5bae2016-04-28 09:23:51 +00003424 const OMPTaskDataTy &Data) {
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003425 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
Alexey Bataev24b5bae2016-04-28 09:23:51 +00003426 SharedsTy, CapturedStruct, IfCond,
3427 Data);
Alexey Bataev7292c292016-04-25 12:22:29 +00003428 };
Alexey Bataev475a7442018-01-12 19:39:11 +00003429 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
Alexey Bataev9c2e8ee2014-07-11 11:25:16 +00003430}
3431
Alexey Bataev9f797f32015-02-05 05:57:51 +00003432void CodeGenFunction::EmitOMPTaskyieldDirective(
3433 const OMPTaskyieldDirective &S) {
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003434 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
Alexey Bataev68446b72014-07-18 07:47:19 +00003435}
3436
Alexey Bataev8f7c1b02014-12-05 04:09:23 +00003437void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003438 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
Alexey Bataev4d1dfea2014-07-18 09:11:51 +00003439}
3440
Alexey Bataev8b8e2022015-04-27 05:22:09 +00003441void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003442 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
Alexey Bataev2df347a2014-07-18 10:17:07 +00003443}
3444
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00003445void CodeGenFunction::EmitOMPTaskgroupDirective(
3446 const OMPTaskgroupDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003447 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3448 Action.Enter(CGF);
Alexey Bataev3b1b8952017-07-25 15:53:26 +00003449 if (const Expr *E = S.getReductionRef()) {
3450 SmallVector<const Expr *, 4> LHSs;
3451 SmallVector<const Expr *, 4> RHSs;
3452 OMPTaskDataTy Data;
3453 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3454 auto IPriv = C->privates().begin();
3455 auto IRed = C->reduction_ops().begin();
3456 auto ILHS = C->lhs_exprs().begin();
3457 auto IRHS = C->rhs_exprs().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00003458 for (const Expr *Ref : C->varlists()) {
Alexey Bataev3b1b8952017-07-25 15:53:26 +00003459 Data.ReductionVars.emplace_back(Ref);
3460 Data.ReductionCopies.emplace_back(*IPriv);
3461 Data.ReductionOps.emplace_back(*IRed);
3462 LHSs.emplace_back(*ILHS);
3463 RHSs.emplace_back(*IRHS);
3464 std::advance(IPriv, 1);
3465 std::advance(IRed, 1);
3466 std::advance(ILHS, 1);
3467 std::advance(IRHS, 1);
3468 }
3469 }
3470 llvm::Value *ReductionDesc =
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003471 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
Alexey Bataev3b1b8952017-07-25 15:53:26 +00003472 LHSs, RHSs, Data);
3473 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3474 CGF.EmitVarDecl(*VD);
3475 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3476 /*Volatile=*/false, E->getType());
3477 }
Alexey Bataev475a7442018-01-12 19:39:11 +00003478 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00003479 };
Alexey Bataev475a7442018-01-12 19:39:11 +00003480 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003481 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
Alexey Bataevc30dd2d2015-06-18 12:14:09 +00003482}
3483
Alexey Bataevcc37cc12014-11-20 04:34:54 +00003484void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00003485 CGM.getOpenMPRuntime().emitFlush(
3486 *this,
3487 [&S]() -> ArrayRef<const Expr *> {
3488 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3489 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3490 FlushClause->varlist_end());
3491 return llvm::None;
3492 }(),
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003493 S.getBeginLoc());
Alexey Bataev6125da92014-07-21 11:26:11 +00003494}
3495
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003496void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
3497 const CodeGenLoopTy &CodeGenLoop,
3498 Expr *IncExpr) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003499 // Emit the loop iteration variable.
Alexey Bataevddf3db92018-04-13 17:31:06 +00003500 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3501 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003502 EmitVarDecl(*IVDecl);
3503
3504 // Emit the iterations count variable.
3505 // If it is not a variable, Sema decided to calculate iterations count on each
3506 // iteration (e.g., it is foldable into a constant).
Alexey Bataevddf3db92018-04-13 17:31:06 +00003507 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003508 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3509 // Emit calculation of the iterations count.
3510 EmitIgnoredExpr(S.getCalcLastIteration());
3511 }
3512
Alexey Bataevddf3db92018-04-13 17:31:06 +00003513 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003514
Carlo Bertolli962bb802017-01-03 18:24:42 +00003515 bool HasLastprivateClause = false;
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003516 // Check pre-condition.
3517 {
Alexey Bataev5a3af132016-03-29 08:58:54 +00003518 OMPLoopScope PreInitScope(*this, S);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003519 // Skip the entire loop if we don't meet the precondition.
3520 // If the condition constant folds and can be elided, avoid emitting the
3521 // whole loop.
3522 bool CondConstant;
3523 llvm::BasicBlock *ContBlock = nullptr;
3524 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3525 if (!CondConstant)
3526 return;
3527 } else {
Alexey Bataevddf3db92018-04-13 17:31:06 +00003528 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003529 ContBlock = createBasicBlock("omp.precond.end");
3530 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3531 getProfileCount(&S));
3532 EmitBlock(ThenBlock);
3533 incrementProfileCounter(&S);
3534 }
3535
Alexey Bataev617db5f2017-12-04 15:38:33 +00003536 emitAlignedClause(*this, S);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003537 // Emit 'then' code.
3538 {
3539 // Emit helper vars inits.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003540
3541 LValue LB = EmitOMPHelperVar(
3542 *this, cast<DeclRefExpr>(
3543 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3544 ? S.getCombinedLowerBoundVariable()
3545 : S.getLowerBoundVariable())));
3546 LValue UB = EmitOMPHelperVar(
3547 *this, cast<DeclRefExpr>(
3548 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3549 ? S.getCombinedUpperBoundVariable()
3550 : S.getUpperBoundVariable())));
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003551 LValue ST =
3552 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3553 LValue IL =
3554 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3555
3556 OMPPrivateScope LoopScope(*this);
Carlo Bertolli962bb802017-01-03 18:24:42 +00003557 if (EmitOMPFirstprivateClause(S, LoopScope)) {
Alexey Bataev617db5f2017-12-04 15:38:33 +00003558 // Emit implicit barrier to synchronize threads and avoid data races
3559 // on initialization of firstprivate variables and post-update of
Carlo Bertolli962bb802017-01-03 18:24:42 +00003560 // lastprivate variables.
3561 CGM.getOpenMPRuntime().emitBarrierCall(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003562 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
Alexey Bataev617db5f2017-12-04 15:38:33 +00003563 /*ForceSimpleCall=*/true);
Carlo Bertolli962bb802017-01-03 18:24:42 +00003564 }
3565 EmitOMPPrivateClause(S, LoopScope);
Alexey Bataev617db5f2017-12-04 15:38:33 +00003566 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
Alexey Bataev999277a2017-12-06 14:31:09 +00003567 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3568 !isOpenMPTeamsDirective(S.getDirectiveKind()))
Alexey Bataev617db5f2017-12-04 15:38:33 +00003569 EmitOMPReductionClauseInit(S, LoopScope);
Carlo Bertolli962bb802017-01-03 18:24:42 +00003570 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
Alexey Bataev5dff95c2016-04-22 03:56:56 +00003571 EmitOMPPrivateLoopCounters(S, LoopScope);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003572 (void)LoopScope.Privatize();
Alexey Bataev60705422018-10-30 15:50:12 +00003573 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3574 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003575
3576 // Detect the distribute schedule kind and chunk.
3577 llvm::Value *Chunk = nullptr;
3578 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
Alexey Bataevddf3db92018-04-13 17:31:06 +00003579 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003580 ScheduleKind = C->getDistScheduleKind();
Alexey Bataevddf3db92018-04-13 17:31:06 +00003581 if (const Expr *Ch = C->getChunkSize()) {
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003582 Chunk = EmitScalarExpr(Ch);
3583 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
Alexey Bataev617db5f2017-12-04 15:38:33 +00003584 S.getIterationVariable()->getType(),
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003585 S.getBeginLoc());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003586 }
Gheorghe-Teodor Bercea02650d42018-09-27 19:22:56 +00003587 } else {
3588 // Default behaviour for dist_schedule clause.
3589 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3590 *this, S, ScheduleKind, Chunk);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003591 }
3592 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3593 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3594
3595 // OpenMP [2.10.8, distribute Construct, Description]
3596 // If dist_schedule is specified, kind must be static. If specified,
3597 // iterations are divided into chunks of size chunk_size, chunks are
3598 // assigned to the teams of the league in a round-robin fashion in the
3599 // order of the team number. When no chunk_size is specified, the
3600 // iteration space is divided into chunks that are approximately equal
3601 // in size, and at most one chunk is distributed to each team of the
3602 // league. The size of the chunks is unspecified in this case.
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00003603 bool StaticChunked = RT.isStaticChunked(
3604 ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3605 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003606 if (RT.isStaticNonchunked(ScheduleKind,
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00003607 /* Chunked */ Chunk != nullptr) ||
3608 StaticChunked) {
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003609 CGOpenMPRuntime::StaticRTInput StaticInit(
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003610 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(*this),
3611 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00003612 StaticChunked ? Chunk : nullptr);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003613 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
Alexey Bataev0f87dbe2017-08-14 17:56:13 +00003614 StaticInit);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003615 JumpDest LoopExit =
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003616 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3617 // UB = min(UB, GlobalUB);
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003618 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3619 ? S.getCombinedEnsureUpperBound()
3620 : S.getEnsureUpperBound());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003621 // IV = LB;
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003622 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3623 ? S.getCombinedInit()
3624 : S.getInit());
3625
Alexey Bataevddf3db92018-04-13 17:31:06 +00003626 const Expr *Cond =
3627 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3628 ? S.getCombinedCond()
3629 : S.getCond();
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003630
Gheorghe-Teodor Berceae9256762018-10-29 15:45:47 +00003631 if (StaticChunked)
3632 Cond = S.getCombinedDistCond();
3633
3634 // For static unchunked schedules generate:
3635 //
3636 // 1. For distribute alone, codegen
3637 // while (idx <= UB) {
3638 // BODY;
3639 // ++idx;
3640 // }
3641 //
3642 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3643 // while (idx <= UB) {
3644 // <CodeGen rest of pragma>(LB, UB);
3645 // idx += ST;
3646 // }
3647 //
3648 // For static chunk one schedule generate:
3649 //
3650 // while (IV <= GlobalUB) {
3651 // <CodeGen rest of pragma>(LB, UB);
3652 // LB += ST;
3653 // UB += ST;
3654 // UB = min(UB, GlobalUB);
3655 // IV = LB;
3656 // }
3657 //
Alexey Bataev779a1802019-12-06 12:21:31 -05003658 emitCommonSimdLoop(
3659 *this, S,
3660 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3661 if (isOpenMPSimdDirective(S.getDirectiveKind()))
3662 CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3663 },
3664 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
3665 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
3666 CGF.EmitOMPInnerLoop(
3667 S, LoopScope.requiresCleanups(), Cond, IncExpr,
3668 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3669 CodeGenLoop(CGF, S, LoopExit);
3670 },
3671 [&S, StaticChunked](CodeGenFunction &CGF) {
3672 if (StaticChunked) {
3673 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3674 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3675 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3676 CGF.EmitIgnoredExpr(S.getCombinedInit());
3677 }
3678 });
3679 });
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003680 EmitBlock(LoopExit.getBlock());
3681 // Tell the runtime we are done.
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003682 RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003683 } else {
3684 // Emit the outer loop, which requests its work chunk [LB..UB] from
3685 // runtime and runs the inner loop to process it.
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003686 const OMPLoopArguments LoopArguments = {
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003687 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
3688 IL.getAddress(*this), Chunk};
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003689 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3690 CodeGenLoop);
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003691 }
Alexey Bataev617db5f2017-12-04 15:38:33 +00003692 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00003693 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
Alexey Bataev617db5f2017-12-04 15:38:33 +00003694 return CGF.Builder.CreateIsNotNull(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003695 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
Alexey Bataev617db5f2017-12-04 15:38:33 +00003696 });
3697 }
Carlo Bertollibeda2142018-02-22 19:38:14 +00003698 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3699 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3700 !isOpenMPTeamsDirective(S.getDirectiveKind())) {
Jonas Hahnfeld5aaaece2018-10-02 19:12:47 +00003701 EmitOMPReductionClauseFinal(S, OMPD_simd);
Carlo Bertollibeda2142018-02-22 19:38:14 +00003702 // Emit post-update of the reduction variables if IsLastIter != 0.
3703 emitPostUpdateForReductionClause(
Alexey Bataevddf3db92018-04-13 17:31:06 +00003704 *this, S, [IL, &S](CodeGenFunction &CGF) {
Carlo Bertollibeda2142018-02-22 19:38:14 +00003705 return CGF.Builder.CreateIsNotNull(
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003706 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
Carlo Bertollibeda2142018-02-22 19:38:14 +00003707 });
Alexey Bataev617db5f2017-12-04 15:38:33 +00003708 }
Carlo Bertolli962bb802017-01-03 18:24:42 +00003709 // Emit final copy of the lastprivate variables if IsLastIter != 0.
Alexey Bataev617db5f2017-12-04 15:38:33 +00003710 if (HasLastprivateClause) {
Carlo Bertolli962bb802017-01-03 18:24:42 +00003711 EmitOMPLastprivateClauseFinal(
3712 S, /*NoFinals=*/false,
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003713 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
Alexey Bataev617db5f2017-12-04 15:38:33 +00003714 }
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003715 }
3716
3717 // We're now done with the loop, so jump to the continuation block.
3718 if (ContBlock) {
3719 EmitBranch(ContBlock);
3720 EmitBlock(ContBlock, true);
3721 }
3722 }
3723}
3724
Carlo Bertolli6200a3d2015-12-14 14:51:25 +00003725void CodeGenFunction::EmitOMPDistributeDirective(
3726 const OMPDistributeDirective &S) {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003727 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00003728 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
Carlo Bertollifc35ad22016-03-07 16:04:49 +00003729 };
Alexey Bataev475a7442018-01-12 19:39:11 +00003730 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev10a54312017-11-27 16:54:08 +00003731 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
Carlo Bertolli6200a3d2015-12-14 14:51:25 +00003732}
3733
Alexey Bataev5f600d62015-09-29 03:48:57 +00003734static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3735 const CapturedStmt *S) {
3736 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3737 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3738 CGF.CapturedStmtInfo = &CapStmtInfo;
Alexey Bataevddf3db92018-04-13 17:31:06 +00003739 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
Alexey Bataevc0f879b2018-04-10 20:10:53 +00003740 Fn->setDoesNotRecurse();
Alexey Bataev5f600d62015-09-29 03:48:57 +00003741 return Fn;
3742}
3743
Alexey Bataev98eb6e32015-04-22 11:15:40 +00003744void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
Alexey Bataev475a7442018-01-12 19:39:11 +00003745 if (S.hasClausesOfKind<OMPDependClause>()) {
3746 assert(!S.getAssociatedStmt() &&
3747 "No associated statement must be in ordered depend construct.");
Alexey Bataev8b427062016-05-25 12:36:08 +00003748 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3749 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
Alexey Bataev8ef31412015-12-18 07:58:25 +00003750 return;
Alexey Bataev8b427062016-05-25 12:36:08 +00003751 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00003752 const auto *C = S.getSingleClause<OMPSIMDClause>();
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003753 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3754 PrePostActionTy &Action) {
Alexey Bataev475a7442018-01-12 19:39:11 +00003755 const CapturedStmt *CS = S.getInnermostCapturedStmt();
Alexey Bataev5f600d62015-09-29 03:48:57 +00003756 if (C) {
Alexey Bataev5f600d62015-09-29 03:48:57 +00003757 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3758 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
Alexey Bataevddf3db92018-04-13 17:31:06 +00003759 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003760 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
Alexey Bataev3c595a62017-08-14 15:01:03 +00003761 OutlinedFn, CapturedVars);
Alexey Bataev5f600d62015-09-29 03:48:57 +00003762 } else {
Alexey Bataev14fa1c62016-03-29 05:34:15 +00003763 Action.Enter(CGF);
Alexey Bataev475a7442018-01-12 19:39:11 +00003764 CGF.EmitStmt(CS->getCapturedStmt());
Alexey Bataev5f600d62015-09-29 03:48:57 +00003765 }
Alexey Bataev98eb6e32015-04-22 11:15:40 +00003766 };
Alexey Bataev475a7442018-01-12 19:39:11 +00003767 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00003768 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
Alexey Bataev9fb6e642014-07-22 06:45:04 +00003769}
3770
Alexey Bataevb57056f2015-01-22 06:17:56 +00003771static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003772 QualType SrcType, QualType DestType,
3773 SourceLocation Loc) {
Alexey Bataevb57056f2015-01-22 06:17:56 +00003774 assert(CGF.hasScalarEvaluationKind(DestType) &&
3775 "DestType must have scalar evaluation kind.");
3776 assert(!Val.isAggregate() && "Must be a scalar or complex.");
Alexey Bataevddf3db92018-04-13 17:31:06 +00003777 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3778 DestType, Loc)
3779 : CGF.EmitComplexToScalarConversion(
3780 Val.getComplexVal(), SrcType, DestType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003781}
3782
3783static CodeGenFunction::ComplexPairTy
3784convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003785 QualType DestType, SourceLocation Loc) {
Alexey Bataevb57056f2015-01-22 06:17:56 +00003786 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3787 "DestType must have complex evaluation kind.");
3788 CodeGenFunction::ComplexPairTy ComplexVal;
3789 if (Val.isScalar()) {
3790 // Convert the input element to the element type of the complex.
Alexey Bataevddf3db92018-04-13 17:31:06 +00003791 QualType DestElementType =
3792 DestType->castAs<ComplexType>()->getElementType();
3793 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3794 Val.getScalarVal(), SrcType, DestElementType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003795 ComplexVal = CodeGenFunction::ComplexPairTy(
3796 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3797 } else {
3798 assert(Val.isComplex() && "Must be a scalar or complex.");
Alexey Bataevddf3db92018-04-13 17:31:06 +00003799 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3800 QualType DestElementType =
3801 DestType->castAs<ComplexType>()->getElementType();
Alexey Bataevb57056f2015-01-22 06:17:56 +00003802 ComplexVal.first = CGF.EmitScalarConversion(
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003803 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003804 ComplexVal.second = CGF.EmitScalarConversion(
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00003805 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003806 }
3807 return ComplexVal;
3808}
3809
Alexey Bataev5e018f92015-04-23 06:35:10 +00003810static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3811 LValue LVal, RValue RVal) {
3812 if (LVal.isGlobalReg()) {
3813 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3814 } else {
JF Bastien92f4ef12016-04-06 17:26:42 +00003815 CGF.EmitAtomicStore(RVal, LVal,
3816 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3817 : llvm::AtomicOrdering::Monotonic,
Rui Ueyama49a3ad22019-07-16 04:46:31 +00003818 LVal.isVolatile(), /*isInit=*/false);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003819 }
3820}
3821
Alexey Bataev8524d152016-01-21 12:35:58 +00003822void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3823 QualType RValTy, SourceLocation Loc) {
3824 switch (getEvaluationKind(LVal.getType())) {
Alexey Bataev5e018f92015-04-23 06:35:10 +00003825 case TEK_Scalar:
Alexey Bataev8524d152016-01-21 12:35:58 +00003826 EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3827 *this, RVal, RValTy, LVal.getType(), Loc)),
3828 LVal);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003829 break;
3830 case TEK_Complex:
Alexey Bataev8524d152016-01-21 12:35:58 +00003831 EmitStoreOfComplex(
3832 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
Alexey Bataev5e018f92015-04-23 06:35:10 +00003833 /*isInit=*/false);
3834 break;
3835 case TEK_Aggregate:
3836 llvm_unreachable("Must be a scalar or complex.");
3837 }
3838}
3839
Alexey Bataevddf3db92018-04-13 17:31:06 +00003840static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
Alexey Bataevb57056f2015-01-22 06:17:56 +00003841 const Expr *X, const Expr *V,
3842 SourceLocation Loc) {
3843 // v = x;
3844 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3845 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3846 LValue XLValue = CGF.EmitLValue(X);
3847 LValue VLValue = CGF.EmitLValue(V);
David Majnemera5b195a2015-02-14 01:35:12 +00003848 RValue Res = XLValue.isGlobalReg()
3849 ? CGF.EmitLoadOfLValue(XLValue, Loc)
JF Bastien92f4ef12016-04-06 17:26:42 +00003850 : CGF.EmitAtomicLoad(
3851 XLValue, Loc,
3852 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3853 : llvm::AtomicOrdering::Monotonic,
3854 XLValue.isVolatile());
Alexey Bataevb57056f2015-01-22 06:17:56 +00003855 // OpenMP, 2.12.6, atomic Construct
3856 // Any atomic construct with a seq_cst clause forces the atomically
3857 // performed operation to include an implicit flush operation without a
3858 // list.
3859 if (IsSeqCst)
Alexey Bataev3eff5f42015-02-25 08:32:46 +00003860 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
Alexey Bataev8524d152016-01-21 12:35:58 +00003861 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00003862}
3863
Alexey Bataevddf3db92018-04-13 17:31:06 +00003864static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
Alexey Bataevb8329262015-02-27 06:33:30 +00003865 const Expr *X, const Expr *E,
3866 SourceLocation Loc) {
3867 // x = expr;
3868 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
Alexey Bataev5e018f92015-04-23 06:35:10 +00003869 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
Alexey Bataevb8329262015-02-27 06:33:30 +00003870 // OpenMP, 2.12.6, atomic Construct
3871 // Any atomic construct with a seq_cst clause forces the atomically
3872 // performed operation to include an implicit flush operation without a
3873 // list.
3874 if (IsSeqCst)
3875 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3876}
3877
Benjamin Kramer439ee9d2015-05-01 13:59:53 +00003878static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3879 RValue Update,
3880 BinaryOperatorKind BO,
3881 llvm::AtomicOrdering AO,
3882 bool IsXLHSInRHSPart) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00003883 ASTContext &Context = CGF.getContext();
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003884 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
Alexey Bataevb4505a72015-03-30 05:20:59 +00003885 // expression is simple and atomic is allowed for the given type for the
3886 // target platform.
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003887 if (BO == BO_Comma || !Update.isScalar() ||
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003888 !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() ||
3889 (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3890 (Update.getScalarVal()->getType() !=
3891 X.getAddress(CGF).getElementType())) ||
3892 !X.getAddress(CGF).getElementType()->isIntegerTy() ||
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003893 !Context.getTargetInfo().hasBuiltinAtomic(
3894 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
Alexey Bataev5e018f92015-04-23 06:35:10 +00003895 return std::make_pair(false, RValue::get(nullptr));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003896
3897 llvm::AtomicRMWInst::BinOp RMWOp;
3898 switch (BO) {
3899 case BO_Add:
3900 RMWOp = llvm::AtomicRMWInst::Add;
3901 break;
3902 case BO_Sub:
3903 if (!IsXLHSInRHSPart)
Alexey Bataev5e018f92015-04-23 06:35:10 +00003904 return std::make_pair(false, RValue::get(nullptr));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003905 RMWOp = llvm::AtomicRMWInst::Sub;
3906 break;
3907 case BO_And:
3908 RMWOp = llvm::AtomicRMWInst::And;
3909 break;
3910 case BO_Or:
3911 RMWOp = llvm::AtomicRMWInst::Or;
3912 break;
3913 case BO_Xor:
3914 RMWOp = llvm::AtomicRMWInst::Xor;
3915 break;
3916 case BO_LT:
3917 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3918 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3919 : llvm::AtomicRMWInst::Max)
3920 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3921 : llvm::AtomicRMWInst::UMax);
3922 break;
3923 case BO_GT:
3924 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3925 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3926 : llvm::AtomicRMWInst::Min)
3927 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3928 : llvm::AtomicRMWInst::UMin);
3929 break;
Alexey Bataev5e018f92015-04-23 06:35:10 +00003930 case BO_Assign:
3931 RMWOp = llvm::AtomicRMWInst::Xchg;
3932 break;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003933 case BO_Mul:
3934 case BO_Div:
3935 case BO_Rem:
3936 case BO_Shl:
3937 case BO_Shr:
3938 case BO_LAnd:
3939 case BO_LOr:
Alexey Bataev5e018f92015-04-23 06:35:10 +00003940 return std::make_pair(false, RValue::get(nullptr));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003941 case BO_PtrMemD:
3942 case BO_PtrMemI:
3943 case BO_LE:
3944 case BO_GE:
3945 case BO_EQ:
3946 case BO_NE:
Richard Smithc70f1d62017-12-14 15:16:18 +00003947 case BO_Cmp:
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003948 case BO_AddAssign:
3949 case BO_SubAssign:
3950 case BO_AndAssign:
3951 case BO_OrAssign:
3952 case BO_XorAssign:
3953 case BO_MulAssign:
3954 case BO_DivAssign:
3955 case BO_RemAssign:
3956 case BO_ShlAssign:
3957 case BO_ShrAssign:
3958 case BO_Comma:
3959 llvm_unreachable("Unsupported atomic update operation");
3960 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00003961 llvm::Value *UpdateVal = Update.getScalarVal();
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003962 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3963 UpdateVal = CGF.Builder.CreateIntCast(
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003964 IC, X.getAddress(CGF).getElementType(),
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003965 X.getType()->hasSignedIntegerRepresentation());
3966 }
Alexey Bataevddf3db92018-04-13 17:31:06 +00003967 llvm::Value *Res =
Akira Hatanakaf139ae32019-12-03 15:17:01 -08003968 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO);
Alexey Bataev5e018f92015-04-23 06:35:10 +00003969 return std::make_pair(true, RValue::get(Res));
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003970}
3971
Alexey Bataev5e018f92015-04-23 06:35:10 +00003972std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003973 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3974 llvm::AtomicOrdering AO, SourceLocation Loc,
Alexey Bataevddf3db92018-04-13 17:31:06 +00003975 const llvm::function_ref<RValue(RValue)> CommonGen) {
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003976 // Update expressions are allowed to have the following forms:
3977 // x binop= expr; -> xrval + expr;
3978 // x++, ++x -> xrval + 1;
3979 // x--, --x -> xrval - 1;
3980 // x = x binop expr; -> xrval binop expr
3981 // x = expr Op x; - > expr binop xrval;
Alexey Bataev5e018f92015-04-23 06:35:10 +00003982 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3983 if (!Res.first) {
Alexey Bataev794ba0d2015-04-10 10:43:45 +00003984 if (X.isGlobalReg()) {
3985 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3986 // 'xrval'.
3987 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3988 } else {
3989 // Perform compare-and-swap procedure.
3990 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
Alexey Bataevb4505a72015-03-30 05:20:59 +00003991 }
3992 }
Alexey Bataev5e018f92015-04-23 06:35:10 +00003993 return Res;
Alexey Bataevb4505a72015-03-30 05:20:59 +00003994}
3995
Alexey Bataevddf3db92018-04-13 17:31:06 +00003996static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
Alexey Bataevb4505a72015-03-30 05:20:59 +00003997 const Expr *X, const Expr *E,
3998 const Expr *UE, bool IsXLHSInRHSPart,
3999 SourceLocation Loc) {
4000 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
4001 "Update expr in 'atomic update' must be a binary operator.");
Alexey Bataevddf3db92018-04-13 17:31:06 +00004002 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
Alexey Bataevb4505a72015-03-30 05:20:59 +00004003 // Update expressions are allowed to have the following forms:
4004 // x binop= expr; -> xrval + expr;
4005 // x++, ++x -> xrval + 1;
4006 // x--, --x -> xrval - 1;
4007 // x = x binop expr; -> xrval binop expr
4008 // x = expr Op x; - > expr binop xrval;
Alexey Bataev794ba0d2015-04-10 10:43:45 +00004009 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
Alexey Bataevb4505a72015-03-30 05:20:59 +00004010 LValue XLValue = CGF.EmitLValue(X);
4011 RValue ExprRValue = CGF.EmitAnyExpr(E);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004012 llvm::AtomicOrdering AO = IsSeqCst
4013 ? llvm::AtomicOrdering::SequentiallyConsistent
4014 : llvm::AtomicOrdering::Monotonic;
4015 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
4016 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
4017 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
4018 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
4019 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
4020 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
4021 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
4022 return CGF.EmitAnyExpr(UE);
4023 };
Alexey Bataev5e018f92015-04-23 06:35:10 +00004024 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
4025 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
4026 // OpenMP, 2.12.6, atomic Construct
4027 // Any atomic construct with a seq_cst clause forces the atomically
4028 // performed operation to include an implicit flush operation without a
4029 // list.
4030 if (IsSeqCst)
4031 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
4032}
4033
4034static RValue convertToType(CodeGenFunction &CGF, RValue Value,
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00004035 QualType SourceType, QualType ResType,
4036 SourceLocation Loc) {
Alexey Bataev5e018f92015-04-23 06:35:10 +00004037 switch (CGF.getEvaluationKind(ResType)) {
4038 case TEK_Scalar:
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00004039 return RValue::get(
4040 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
Alexey Bataev5e018f92015-04-23 06:35:10 +00004041 case TEK_Complex: {
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00004042 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
Alexey Bataev5e018f92015-04-23 06:35:10 +00004043 return RValue::getComplex(Res.first, Res.second);
4044 }
4045 case TEK_Aggregate:
4046 break;
4047 }
4048 llvm_unreachable("Must be a scalar or complex.");
4049}
4050
Alexey Bataevddf3db92018-04-13 17:31:06 +00004051static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
Alexey Bataev5e018f92015-04-23 06:35:10 +00004052 bool IsPostfixUpdate, const Expr *V,
4053 const Expr *X, const Expr *E,
4054 const Expr *UE, bool IsXLHSInRHSPart,
4055 SourceLocation Loc) {
4056 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
4057 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
4058 RValue NewVVal;
4059 LValue VLValue = CGF.EmitLValue(V);
4060 LValue XLValue = CGF.EmitLValue(X);
4061 RValue ExprRValue = CGF.EmitAnyExpr(E);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004062 llvm::AtomicOrdering AO = IsSeqCst
4063 ? llvm::AtomicOrdering::SequentiallyConsistent
4064 : llvm::AtomicOrdering::Monotonic;
Alexey Bataev5e018f92015-04-23 06:35:10 +00004065 QualType NewVValType;
4066 if (UE) {
4067 // 'x' is updated with some additional value.
4068 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
4069 "Update expr in 'atomic capture' must be a binary operator.");
Alexey Bataevddf3db92018-04-13 17:31:06 +00004070 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
Alexey Bataev5e018f92015-04-23 06:35:10 +00004071 // Update expressions are allowed to have the following forms:
4072 // x binop= expr; -> xrval + expr;
4073 // x++, ++x -> xrval + 1;
4074 // x--, --x -> xrval - 1;
4075 // x = x binop expr; -> xrval binop expr
4076 // x = expr Op x; - > expr binop xrval;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004077 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
4078 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
4079 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
Alexey Bataev5e018f92015-04-23 06:35:10 +00004080 NewVValType = XRValExpr->getType();
Alexey Bataevddf3db92018-04-13 17:31:06 +00004081 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
Alexey Bataev5e018f92015-04-23 06:35:10 +00004082 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
Alexey Bataevddf3db92018-04-13 17:31:06 +00004083 IsPostfixUpdate](RValue XRValue) {
Alexey Bataev5e018f92015-04-23 06:35:10 +00004084 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
4085 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
4086 RValue Res = CGF.EmitAnyExpr(UE);
4087 NewVVal = IsPostfixUpdate ? XRValue : Res;
4088 return Res;
4089 };
4090 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
4091 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
4092 if (Res.first) {
4093 // 'atomicrmw' instruction was generated.
4094 if (IsPostfixUpdate) {
4095 // Use old value from 'atomicrmw'.
4096 NewVVal = Res.second;
4097 } else {
4098 // 'atomicrmw' does not provide new value, so evaluate it using old
4099 // value of 'x'.
4100 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
4101 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
4102 NewVVal = CGF.EmitAnyExpr(UE);
4103 }
4104 }
4105 } else {
4106 // 'x' is simply rewritten with some 'expr'.
4107 NewVValType = X->getType().getNonReferenceType();
4108 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
Filipe Cabecinhas7af183d2015-08-11 04:19:28 +00004109 X->getType().getNonReferenceType(), Loc);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004110 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
Alexey Bataev5e018f92015-04-23 06:35:10 +00004111 NewVVal = XRValue;
4112 return ExprRValue;
4113 };
4114 // Try to perform atomicrmw xchg, otherwise simple exchange.
4115 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
4116 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
4117 Loc, Gen);
4118 if (Res.first) {
4119 // 'atomicrmw' instruction was generated.
4120 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
4121 }
4122 }
4123 // Emit post-update store to 'v' of old/new 'x' value.
Alexey Bataev8524d152016-01-21 12:35:58 +00004124 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
Alexey Bataevb4505a72015-03-30 05:20:59 +00004125 // OpenMP, 2.12.6, atomic Construct
4126 // Any atomic construct with a seq_cst clause forces the atomically
4127 // performed operation to include an implicit flush operation without a
4128 // list.
4129 if (IsSeqCst)
4130 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
4131}
4132
Alexey Bataevddf3db92018-04-13 17:31:06 +00004133static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
Alexey Bataev5e018f92015-04-23 06:35:10 +00004134 bool IsSeqCst, bool IsPostfixUpdate,
4135 const Expr *X, const Expr *V, const Expr *E,
4136 const Expr *UE, bool IsXLHSInRHSPart,
4137 SourceLocation Loc) {
Alexey Bataevb57056f2015-01-22 06:17:56 +00004138 switch (Kind) {
4139 case OMPC_read:
Alexey Bataevddf3db92018-04-13 17:31:06 +00004140 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
Alexey Bataevb57056f2015-01-22 06:17:56 +00004141 break;
4142 case OMPC_write:
Alexey Bataevddf3db92018-04-13 17:31:06 +00004143 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
Alexey Bataevb8329262015-02-27 06:33:30 +00004144 break;
Alexey Bataevb4505a72015-03-30 05:20:59 +00004145 case OMPC_unknown:
Alexey Bataevb57056f2015-01-22 06:17:56 +00004146 case OMPC_update:
Alexey Bataevddf3db92018-04-13 17:31:06 +00004147 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
Alexey Bataevb4505a72015-03-30 05:20:59 +00004148 break;
Alexey Bataevb57056f2015-01-22 06:17:56 +00004149 case OMPC_capture:
Alexey Bataevddf3db92018-04-13 17:31:06 +00004150 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
Alexey Bataev5e018f92015-04-23 06:35:10 +00004151 IsXLHSInRHSPart, Loc);
4152 break;
Alexey Bataevb57056f2015-01-22 06:17:56 +00004153 case OMPC_if:
4154 case OMPC_final:
4155 case OMPC_num_threads:
4156 case OMPC_private:
4157 case OMPC_firstprivate:
4158 case OMPC_lastprivate:
4159 case OMPC_reduction:
Alexey Bataev169d96a2017-07-18 20:17:46 +00004160 case OMPC_task_reduction:
Alexey Bataevfa312f32017-07-21 18:48:21 +00004161 case OMPC_in_reduction:
Alexey Bataevb57056f2015-01-22 06:17:56 +00004162 case OMPC_safelen:
Alexey Bataev66b15b52015-08-21 11:14:16 +00004163 case OMPC_simdlen:
Alexey Bataev9cc10fc2019-03-12 18:52:33 +00004164 case OMPC_allocator:
Alexey Bataeve04483e2019-03-27 14:14:31 +00004165 case OMPC_allocate:
Alexey Bataevb57056f2015-01-22 06:17:56 +00004166 case OMPC_collapse:
4167 case OMPC_default:
4168 case OMPC_seq_cst:
4169 case OMPC_shared:
4170 case OMPC_linear:
4171 case OMPC_aligned:
4172 case OMPC_copyin:
4173 case OMPC_copyprivate:
4174 case OMPC_flush:
4175 case OMPC_proc_bind:
4176 case OMPC_schedule:
4177 case OMPC_ordered:
4178 case OMPC_nowait:
4179 case OMPC_untied:
4180 case OMPC_threadprivate:
Alexey Bataev1c2cfbc2015-06-23 14:25:19 +00004181 case OMPC_depend:
Alexey Bataevb57056f2015-01-22 06:17:56 +00004182 case OMPC_mergeable:
Michael Wonge710d542015-08-07 16:16:36 +00004183 case OMPC_device:
Alexey Bataev346265e2015-09-25 10:37:12 +00004184 case OMPC_threads:
Alexey Bataevd14d1e62015-09-28 06:39:35 +00004185 case OMPC_simd:
Kelvin Li0bff7af2015-11-23 05:32:03 +00004186 case OMPC_map:
Kelvin Li099bb8c2015-11-24 20:50:12 +00004187 case OMPC_num_teams:
Kelvin Lia15fb1a2015-11-27 18:47:36 +00004188 case OMPC_thread_limit:
Alexey Bataeva0569352015-12-01 10:17:31 +00004189 case OMPC_priority:
Alexey Bataev1fd4aed2015-12-07 12:52:51 +00004190 case OMPC_grainsize:
Alexey Bataevb825de12015-12-07 10:51:44 +00004191 case OMPC_nogroup:
Alexey Bataev382967a2015-12-08 12:06:20 +00004192 case OMPC_num_tasks:
Alexey Bataev28c75412015-12-15 08:19:24 +00004193 case OMPC_hint:
Carlo Bertollib4adf552016-01-15 18:50:31 +00004194 case OMPC_dist_schedule:
Arpith Chacko Jacob3cf89042016-01-26 16:37:23 +00004195 case OMPC_defaultmap:
Alexey Bataeve48a5fc2016-04-12 05:28:34 +00004196 case OMPC_uniform:
Samuel Antao661c0902016-05-26 17:39:58 +00004197 case OMPC_to:
Samuel Antaoec172c62016-05-26 17:49:04 +00004198 case OMPC_from:
Carlo Bertolli2404b172016-07-13 15:37:16 +00004199 case OMPC_use_device_ptr:
Carlo Bertolli70594e92016-07-13 17:16:49 +00004200 case OMPC_is_device_ptr:
Kelvin Li1408f912018-09-26 04:28:39 +00004201 case OMPC_unified_address:
Alexey Bataev94c50642018-10-01 14:26:31 +00004202 case OMPC_unified_shared_memory:
Patrick Lyster6bdf63b2018-10-03 20:07:58 +00004203 case OMPC_reverse_offload:
Patrick Lyster3fe9e392018-10-11 14:41:10 +00004204 case OMPC_dynamic_allocators:
Patrick Lyster7a2a27c2018-11-02 12:18:11 +00004205 case OMPC_atomic_default_mem_order:
Alexey Bataev729e2422019-08-23 16:11:14 +00004206 case OMPC_device_type:
Alexey Bataevdba792c2019-09-23 18:13:31 +00004207 case OMPC_match:
Alexey Bataevb57056f2015-01-22 06:17:56 +00004208 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
4209 }
4210}
4211
4212void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
Benjamin Kramerfc600dc2015-08-30 15:12:28 +00004213 bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
Alexey Bataevb57056f2015-01-22 06:17:56 +00004214 OpenMPClauseKind Kind = OMPC_unknown;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004215 for (const OMPClause *C : S.clauses()) {
Alexey Bataevb57056f2015-01-22 06:17:56 +00004216 // Find first clause (skip seq_cst clause, if it is first).
4217 if (C->getClauseKind() != OMPC_seq_cst) {
4218 Kind = C->getClauseKind();
4219 break;
4220 }
4221 }
Alexey Bataev10fec572015-03-11 04:48:56 +00004222
Alexey Bataevddf3db92018-04-13 17:31:06 +00004223 const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
Bill Wendling7c44da22018-10-31 03:48:47 +00004224 if (const auto *FE = dyn_cast<FullExpr>(CS))
4225 enterFullExpression(FE);
Alexey Bataev5e018f92015-04-23 06:35:10 +00004226 // Processing for statements under 'atomic capture'.
4227 if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00004228 for (const Stmt *C : Compound->body()) {
Bill Wendling7c44da22018-10-31 03:48:47 +00004229 if (const auto *FE = dyn_cast<FullExpr>(C))
4230 enterFullExpression(FE);
Alexey Bataev5e018f92015-04-23 06:35:10 +00004231 }
4232 }
Alexey Bataev10fec572015-03-11 04:48:56 +00004233
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004234 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4235 PrePostActionTy &) {
Alexey Bataev33c56402015-12-14 09:26:19 +00004236 CGF.EmitStopPoint(CS);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004237 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
Alexey Bataev5e018f92015-04-23 06:35:10 +00004238 S.getV(), S.getExpr(), S.getUpdateExpr(),
Stephen Kellyf2ceec42018-08-09 21:08:08 +00004239 S.isXLHSInRHSPart(), S.getBeginLoc());
Alexey Bataev6f1ffc02015-04-10 04:50:10 +00004240 };
Alexey Bataev475a7442018-01-12 19:39:11 +00004241 OMPLexicalScope Scope(*this, S, OMPD_unknown);
Alexey Bataev81c7ea02015-07-03 09:56:58 +00004242 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
Alexey Bataev0162e452014-07-22 10:10:35 +00004243}
4244
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004245static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
4246 const OMPExecutableDirective &S,
4247 const RegionCodeGenTy &CodeGen) {
4248 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
4249 CodeGenModule &CGM = CGF.CGM;
Samuel Antaobed3c462015-10-02 16:14:20 +00004250
Alexey Bataev4f4bf7c2018-03-15 15:47:20 +00004251 // On device emit this construct as inlined code.
4252 if (CGM.getLangOpts().OpenMPIsDevice) {
4253 OMPLexicalScope Scope(CGF, S, OMPD_target);
4254 CGM.getOpenMPRuntime().emitInlinedDirective(
4255 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev4ac68a22018-05-16 15:08:32 +00004256 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
Alexey Bataev4f4bf7c2018-03-15 15:47:20 +00004257 });
4258 return;
4259 }
4260
Samuel Antaoee8fb302016-01-06 13:42:12 +00004261 llvm::Function *Fn = nullptr;
4262 llvm::Constant *FnID = nullptr;
Samuel Antaobed3c462015-10-02 16:14:20 +00004263
Samuel Antaobed3c462015-10-02 16:14:20 +00004264 const Expr *IfCond = nullptr;
Arpith Chacko Jacobfe4890a2017-01-18 20:40:48 +00004265 // Check for the at most one if clause associated with the target region.
4266 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4267 if (C->getNameModifier() == OMPD_unknown ||
4268 C->getNameModifier() == OMPD_target) {
4269 IfCond = C->getCondition();
4270 break;
4271 }
Samuel Antaobed3c462015-10-02 16:14:20 +00004272 }
4273
4274 // Check if we have any device clause associated with the directive.
4275 const Expr *Device = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004276 if (auto *C = S.getSingleClause<OMPDeviceClause>())
Samuel Antaobed3c462015-10-02 16:14:20 +00004277 Device = C->getDevice();
Samuel Antaobed3c462015-10-02 16:14:20 +00004278
Samuel Antaoee8fb302016-01-06 13:42:12 +00004279 // Check if we have an if clause whose conditional always evaluates to false
4280 // or if we do not have any targets specified. If so the target region is not
4281 // an offload entry point.
4282 bool IsOffloadEntry = true;
4283 if (IfCond) {
4284 bool Val;
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004285 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
Samuel Antaoee8fb302016-01-06 13:42:12 +00004286 IsOffloadEntry = false;
4287 }
4288 if (CGM.getLangOpts().OMPTargetTriples.empty())
4289 IsOffloadEntry = false;
4290
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004291 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
Samuel Antaoee8fb302016-01-06 13:42:12 +00004292 StringRef ParentName;
4293 // In case we have Ctors/Dtors we use the complete type variant to produce
4294 // the mangling of the device outlined kernel.
Alexey Bataevddf3db92018-04-13 17:31:06 +00004295 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
Samuel Antaoee8fb302016-01-06 13:42:12 +00004296 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
Alexey Bataevddf3db92018-04-13 17:31:06 +00004297 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
Samuel Antaoee8fb302016-01-06 13:42:12 +00004298 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4299 else
4300 ParentName =
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004301 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
Samuel Antaoee8fb302016-01-06 13:42:12 +00004302
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004303 // Emit target region as a standalone region.
4304 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4305 IsOffloadEntry, CodeGen);
Alexey Bataev8451efa2018-01-15 19:06:12 +00004306 OMPLexicalScope Scope(CGF, S, OMPD_task);
Alexey Bataevec7946e2019-09-23 14:06:51 +00004307 auto &&SizeEmitter =
4308 [IsOffloadEntry](CodeGenFunction &CGF,
4309 const OMPLoopDirective &D) -> llvm::Value * {
4310 if (IsOffloadEntry) {
4311 OMPLoopScope(CGF, D);
4312 // Emit calculation of the iterations count.
4313 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4314 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4315 /*isSigned=*/false);
4316 return NumIterations;
4317 }
4318 return nullptr;
Alexey Bataev7bb33532019-01-07 21:30:43 +00004319 };
Alexey Bataevec7946e2019-09-23 14:06:51 +00004320 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
4321 SizeEmitter);
Alexey Bataev0bd520b2014-09-19 08:19:49 +00004322}
4323
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004324static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
4325 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004326 Action.Enter(CGF);
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004327 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4328 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4329 CGF.EmitOMPPrivateClause(S, PrivateScope);
4330 (void)PrivateScope.Privatize();
Alexey Bataev60705422018-10-30 15:50:12 +00004331 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4332 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004333
Alexey Bataev475a7442018-01-12 19:39:11 +00004334 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
Arpith Chacko Jacob43a8b7b2017-01-16 15:26:02 +00004335}
4336
4337void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4338 StringRef ParentName,
4339 const OMPTargetDirective &S) {
4340 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4341 emitTargetRegion(CGF, S, Action);
4342 };
4343 llvm::Function *Fn;
4344 llvm::Constant *Addr;
4345 // Emit target region as a standalone region.
4346 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4347 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4348 assert(Fn && Addr && "Target device function emission failed.");
4349}
4350
4351void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
4352 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4353 emitTargetRegion(CGF, S, Action);
4354 };
4355 emitCommonOMPTargetDirective(*this, S, CodeGen);
4356}
4357
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004358static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
4359 const OMPExecutableDirective &S,
4360 OpenMPDirectiveKind InnermostKind,
4361 const RegionCodeGenTy &CodeGen) {
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004362 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
James Y Knight9871db02019-02-05 16:42:33 +00004363 llvm::Function *OutlinedFn =
Alexey Bataevddf3db92018-04-13 17:31:06 +00004364 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4365 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
Samuel Antaob68e2db2016-03-03 16:20:23 +00004366
Alexey Bataevddf3db92018-04-13 17:31:06 +00004367 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4368 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004369 if (NT || TL) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00004370 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4371 const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004372
Carlo Bertollic6872252016-04-04 15:55:02 +00004373 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
Stephen Kellyf2ceec42018-08-09 21:08:08 +00004374 S.getBeginLoc());
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004375 }
4376
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00004377 OMPTeamsScope Scope(CGF, S);
Alexey Bataev14fa1c62016-03-29 05:34:15 +00004378 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
4379 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00004380 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004381 CapturedVars);
4382}
4383
4384void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
Kelvin Li51336dd2016-12-15 17:55:32 +00004385 // Emit teams region as a standalone region.
Alexey Bataevc99042b2018-03-15 18:10:54 +00004386 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004387 Action.Enter(CGF);
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004388 OMPPrivateScope PrivateScope(CGF);
Carlo Bertolli6ad7b5a2016-03-03 22:09:40 +00004389 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4390 CGF.EmitOMPPrivateClause(S, PrivateScope);
Arpith Chacko Jacobfc711b12017-02-16 16:48:49 +00004391 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004392 (void)PrivateScope.Privatize();
Alexey Bataev475a7442018-01-12 19:39:11 +00004393 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
Arpith Chacko Jacobfc711b12017-02-16 16:48:49 +00004394 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
Carlo Bertolli430d8ec2016-03-03 20:34:23 +00004395 };
Alexey Bataev2139ed62017-11-16 18:20:21 +00004396 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004397 emitPostUpdateForReductionClause(*this, S,
4398 [](CodeGenFunction &) { return nullptr; });
Alexey Bataev13314bf2014-10-09 04:18:56 +00004399}
Alexey Bataev6d4ed052015-07-01 06:57:41 +00004400
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00004401static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4402 const OMPTargetTeamsDirective &S) {
4403 auto *CS = S.getCapturedStmt(OMPD_teams);
4404 Action.Enter(CGF);
Alexey Bataevf9fc42e2017-11-22 14:25:55 +00004405 // Emit teams region as a standalone region.
4406 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004407 Action.Enter(CGF);
Alexey Bataevf9fc42e2017-11-22 14:25:55 +00004408 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4409 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4410 CGF.EmitOMPPrivateClause(S, PrivateScope);
4411 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4412 (void)PrivateScope.Privatize();
Alexey Bataev60705422018-10-30 15:50:12 +00004413 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4414 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00004415 CGF.EmitStmt(CS->getCapturedStmt());
Alexey Bataevf9fc42e2017-11-22 14:25:55 +00004416 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00004417 };
4418 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004419 emitPostUpdateForReductionClause(CGF, S,
4420 [](CodeGenFunction &) { return nullptr; });
Arpith Chacko Jacob99a1e0e2017-01-25 02:18:43 +00004421}
4422
4423void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
4424 CodeGenModule &CGM, StringRef ParentName,
4425 const OMPTargetTeamsDirective &S) {
4426 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4427 emitTargetTeamsRegion(CGF, Action, S);
4428 };
4429 llvm::Function *Fn;
4430 llvm::Constant *Addr;
4431 // Emit target region as a standalone region.
4432 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4433 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4434 assert(Fn && Addr && "Target device function emission failed.");
4435}
4436
4437void CodeGenFunction::EmitOMPTargetTeamsDirective(
4438 const OMPTargetTeamsDirective &S) {
4439 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4440 emitTargetTeamsRegion(CGF, Action, S);
4441 };
4442 emitCommonOMPTargetDirective(*this, S, CodeGen);
4443}
4444
Alexey Bataevdfa430f2017-12-08 15:03:50 +00004445static void
4446emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4447 const OMPTargetTeamsDistributeDirective &S) {
4448 Action.Enter(CGF);
4449 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4450 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4451 };
4452
4453 // Emit teams region as a standalone region.
4454 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004455 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004456 Action.Enter(CGF);
Alexey Bataevdfa430f2017-12-08 15:03:50 +00004457 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4458 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4459 (void)PrivateScope.Privatize();
4460 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4461 CodeGenDistribute);
4462 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4463 };
4464 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4465 emitPostUpdateForReductionClause(CGF, S,
4466 [](CodeGenFunction &) { return nullptr; });
4467}
4468
4469void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
4470 CodeGenModule &CGM, StringRef ParentName,
4471 const OMPTargetTeamsDistributeDirective &S) {
4472 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4473 emitTargetTeamsDistributeRegion(CGF, Action, S);
4474 };
4475 llvm::Function *Fn;
4476 llvm::Constant *Addr;
4477 // Emit target region as a standalone region.
4478 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4479 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4480 assert(Fn && Addr && "Target device function emission failed.");
4481}
4482
4483void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
4484 const OMPTargetTeamsDistributeDirective &S) {
4485 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4486 emitTargetTeamsDistributeRegion(CGF, Action, S);
4487 };
4488 emitCommonOMPTargetDirective(*this, S, CodeGen);
4489}
4490
Alexey Bataevfbe17fb2017-12-13 19:45:06 +00004491static void emitTargetTeamsDistributeSimdRegion(
4492 CodeGenFunction &CGF, PrePostActionTy &Action,
4493 const OMPTargetTeamsDistributeSimdDirective &S) {
4494 Action.Enter(CGF);
4495 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4496 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4497 };
4498
4499 // Emit teams region as a standalone region.
4500 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004501 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004502 Action.Enter(CGF);
Alexey Bataevfbe17fb2017-12-13 19:45:06 +00004503 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4504 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4505 (void)PrivateScope.Privatize();
4506 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4507 CodeGenDistribute);
4508 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4509 };
4510 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4511 emitPostUpdateForReductionClause(CGF, S,
4512 [](CodeGenFunction &) { return nullptr; });
4513}
4514
4515void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4516 CodeGenModule &CGM, StringRef ParentName,
4517 const OMPTargetTeamsDistributeSimdDirective &S) {
4518 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4519 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4520 };
4521 llvm::Function *Fn;
4522 llvm::Constant *Addr;
4523 // Emit target region as a standalone region.
4524 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4525 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4526 assert(Fn && Addr && "Target device function emission failed.");
4527}
4528
4529void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
4530 const OMPTargetTeamsDistributeSimdDirective &S) {
4531 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4532 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4533 };
4534 emitCommonOMPTargetDirective(*this, S, CodeGen);
4535}
4536
Carlo Bertolliba1487b2017-10-04 14:12:09 +00004537void CodeGenFunction::EmitOMPTeamsDistributeDirective(
4538 const OMPTeamsDistributeDirective &S) {
4539
4540 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4541 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4542 };
4543
4544 // Emit teams region as a standalone region.
4545 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004546 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004547 Action.Enter(CGF);
Carlo Bertolliba1487b2017-10-04 14:12:09 +00004548 OMPPrivateScope PrivateScope(CGF);
4549 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4550 (void)PrivateScope.Privatize();
4551 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4552 CodeGenDistribute);
4553 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4554 };
Alexey Bataev95c6dd42017-11-29 15:14:16 +00004555 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
Carlo Bertolliba1487b2017-10-04 14:12:09 +00004556 emitPostUpdateForReductionClause(*this, S,
4557 [](CodeGenFunction &) { return nullptr; });
4558}
4559
Alexey Bataev999277a2017-12-06 14:31:09 +00004560void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
4561 const OMPTeamsDistributeSimdDirective &S) {
4562 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4563 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4564 };
4565
4566 // Emit teams region as a standalone region.
4567 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004568 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004569 Action.Enter(CGF);
Alexey Bataev999277a2017-12-06 14:31:09 +00004570 OMPPrivateScope PrivateScope(CGF);
4571 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4572 (void)PrivateScope.Privatize();
4573 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4574 CodeGenDistribute);
4575 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4576 };
4577 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4578 emitPostUpdateForReductionClause(*this, S,
4579 [](CodeGenFunction &) { return nullptr; });
4580}
4581
Carlo Bertolli62fae152017-11-20 20:46:39 +00004582void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
4583 const OMPTeamsDistributeParallelForDirective &S) {
4584 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4585 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4586 S.getDistInc());
4587 };
4588
4589 // Emit teams region as a standalone region.
4590 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004591 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004592 Action.Enter(CGF);
Carlo Bertolli62fae152017-11-20 20:46:39 +00004593 OMPPrivateScope PrivateScope(CGF);
4594 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4595 (void)PrivateScope.Privatize();
Alexey Bataev10a54312017-11-27 16:54:08 +00004596 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4597 CodeGenDistribute);
Carlo Bertolli62fae152017-11-20 20:46:39 +00004598 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4599 };
4600 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4601 emitPostUpdateForReductionClause(*this, S,
4602 [](CodeGenFunction &) { return nullptr; });
4603}
4604
Carlo Bertolli56a2aa42017-12-04 20:57:19 +00004605void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
4606 const OMPTeamsDistributeParallelForSimdDirective &S) {
4607 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4608 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4609 S.getDistInc());
4610 };
4611
4612 // Emit teams region as a standalone region.
4613 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004614 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004615 Action.Enter(CGF);
Carlo Bertolli56a2aa42017-12-04 20:57:19 +00004616 OMPPrivateScope PrivateScope(CGF);
4617 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4618 (void)PrivateScope.Privatize();
4619 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4620 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4621 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4622 };
4623 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4624 emitPostUpdateForReductionClause(*this, S,
4625 [](CodeGenFunction &) { return nullptr; });
4626}
4627
Carlo Bertolli52978c32018-01-03 21:12:44 +00004628static void emitTargetTeamsDistributeParallelForRegion(
4629 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
4630 PrePostActionTy &Action) {
Carlo Bertolli79712092018-02-28 20:48:35 +00004631 Action.Enter(CGF);
Carlo Bertolli52978c32018-01-03 21:12:44 +00004632 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4633 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4634 S.getDistInc());
4635 };
4636
4637 // Emit teams region as a standalone region.
4638 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004639 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004640 Action.Enter(CGF);
Carlo Bertolli52978c32018-01-03 21:12:44 +00004641 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4642 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4643 (void)PrivateScope.Privatize();
4644 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4645 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4646 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4647 };
4648
4649 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4650 CodeGenTeams);
4651 emitPostUpdateForReductionClause(CGF, S,
4652 [](CodeGenFunction &) { return nullptr; });
4653}
4654
4655void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4656 CodeGenModule &CGM, StringRef ParentName,
4657 const OMPTargetTeamsDistributeParallelForDirective &S) {
4658 // Emit SPMD target teams distribute parallel for region as a standalone
4659 // region.
4660 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4661 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4662 };
4663 llvm::Function *Fn;
4664 llvm::Constant *Addr;
4665 // Emit target region as a standalone region.
4666 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4667 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4668 assert(Fn && Addr && "Target device function emission failed.");
4669}
4670
4671void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
4672 const OMPTargetTeamsDistributeParallelForDirective &S) {
4673 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4674 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4675 };
4676 emitCommonOMPTargetDirective(*this, S, CodeGen);
4677}
4678
Alexey Bataev647dd842018-01-15 20:59:40 +00004679static void emitTargetTeamsDistributeParallelForSimdRegion(
4680 CodeGenFunction &CGF,
4681 const OMPTargetTeamsDistributeParallelForSimdDirective &S,
4682 PrePostActionTy &Action) {
Carlo Bertolli79712092018-02-28 20:48:35 +00004683 Action.Enter(CGF);
Alexey Bataev647dd842018-01-15 20:59:40 +00004684 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4685 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4686 S.getDistInc());
4687 };
4688
4689 // Emit teams region as a standalone region.
4690 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
Alexey Bataevc99042b2018-03-15 18:10:54 +00004691 PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004692 Action.Enter(CGF);
Alexey Bataev647dd842018-01-15 20:59:40 +00004693 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4694 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4695 (void)PrivateScope.Privatize();
4696 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4697 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4698 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4699 };
4700
4701 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4702 CodeGenTeams);
4703 emitPostUpdateForReductionClause(CGF, S,
4704 [](CodeGenFunction &) { return nullptr; });
4705}
4706
4707void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4708 CodeGenModule &CGM, StringRef ParentName,
4709 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4710 // Emit SPMD target teams distribute parallel for simd region as a standalone
4711 // region.
4712 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4713 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4714 };
4715 llvm::Function *Fn;
4716 llvm::Constant *Addr;
4717 // Emit target region as a standalone region.
4718 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4719 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4720 assert(Fn && Addr && "Target device function emission failed.");
4721}
4722
4723void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4724 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4725 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4726 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4727 };
4728 emitCommonOMPTargetDirective(*this, S, CodeGen);
4729}
4730
Alexey Bataev6d4ed052015-07-01 06:57:41 +00004731void CodeGenFunction::EmitOMPCancellationPointDirective(
4732 const OMPCancellationPointDirective &S) {
Stephen Kellyf2ceec42018-08-09 21:08:08 +00004733 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
Alexey Bataev0f34da12015-07-02 04:17:07 +00004734 S.getCancelRegion());
Alexey Bataev6d4ed052015-07-01 06:57:41 +00004735}
4736
Alexey Bataev80909872015-07-02 11:25:17 +00004737void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
Alexey Bataev87933c72015-09-18 08:07:34 +00004738 const Expr *IfCond = nullptr;
4739 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4740 if (C->getNameModifier() == OMPD_unknown ||
4741 C->getNameModifier() == OMPD_cancel) {
4742 IfCond = C->getCondition();
4743 break;
4744 }
4745 }
Stephen Kellyf2ceec42018-08-09 21:08:08 +00004746 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
Alexey Bataev7d5d33e2015-07-06 05:50:32 +00004747 S.getCancelRegion());
Alexey Bataev80909872015-07-02 11:25:17 +00004748}
4749
Alexey Bataev81c7ea02015-07-03 09:56:58 +00004750CodeGenFunction::JumpDest
4751CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
Alexey Bataev957d8562016-11-17 15:12:05 +00004752 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4753 Kind == OMPD_target_parallel)
Alexey Bataev81c7ea02015-07-03 09:56:58 +00004754 return ReturnBlock;
Alexey Bataev25e5b442015-09-15 12:52:43 +00004755 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
Alexey Bataev957d8562016-11-17 15:12:05 +00004756 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4757 Kind == OMPD_distribute_parallel_for ||
Alexey Bataevdcb4b8fb2017-11-22 20:19:50 +00004758 Kind == OMPD_target_parallel_for ||
Alexey Bataev16e79882017-11-22 21:12:03 +00004759 Kind == OMPD_teams_distribute_parallel_for ||
4760 Kind == OMPD_target_teams_distribute_parallel_for);
Alexey Bataev957d8562016-11-17 15:12:05 +00004761 return OMPCancelStack.getExitBlock();
Alexey Bataev81c7ea02015-07-03 09:56:58 +00004762}
Michael Wong65f367f2015-07-21 13:44:28 +00004763
Samuel Antaocc10b852016-07-28 14:23:26 +00004764void CodeGenFunction::EmitOMPUseDevicePtrClause(
4765 const OMPClause &NC, OMPPrivateScope &PrivateScope,
4766 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4767 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4768 auto OrigVarIt = C.varlist_begin();
4769 auto InitIt = C.inits().begin();
Alexey Bataevddf3db92018-04-13 17:31:06 +00004770 for (const Expr *PvtVarIt : C.private_copies()) {
4771 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4772 const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4773 const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
Samuel Antaocc10b852016-07-28 14:23:26 +00004774
4775 // In order to identify the right initializer we need to match the
4776 // declaration used by the mapping logic. In some cases we may get
4777 // OMPCapturedExprDecl that refers to the original declaration.
4778 const ValueDecl *MatchingVD = OrigVD;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004779 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
Samuel Antaocc10b852016-07-28 14:23:26 +00004780 // OMPCapturedExprDecl are used to privative fields of the current
4781 // structure.
Alexey Bataevddf3db92018-04-13 17:31:06 +00004782 const auto *ME = cast<MemberExpr>(OED->getInit());
Samuel Antaocc10b852016-07-28 14:23:26 +00004783 assert(isa<CXXThisExpr>(ME->getBase()) &&
4784 "Base should be the current struct!");
4785 MatchingVD = ME->getMemberDecl();
4786 }
4787
4788 // If we don't have information about the current list item, move on to
4789 // the next one.
4790 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4791 if (InitAddrIt == CaptureDeviceAddrMap.end())
4792 continue;
4793
Alexey Bataevddf3db92018-04-13 17:31:06 +00004794 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4795 InitAddrIt, InitVD,
4796 PvtVD]() {
Samuel Antaocc10b852016-07-28 14:23:26 +00004797 // Initialize the temporary initialization variable with the address we
4798 // get from the runtime library. We have to cast the source address
4799 // because it is always a void *. References are materialized in the
4800 // privatization scope, so the initialization here disregards the fact
4801 // the original variable is a reference.
4802 QualType AddrQTy =
4803 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4804 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4805 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4806 setAddrOfLocalVar(InitVD, InitAddr);
4807
4808 // Emit private declaration, it will be initialized by the value we
4809 // declaration we just added to the local declarations map.
4810 EmitDecl(*PvtVD);
4811
4812 // The initialization variables reached its purpose in the emission
Alexander Kornienko2a8c18d2018-04-06 15:14:32 +00004813 // of the previous declaration, so we don't need it anymore.
Samuel Antaocc10b852016-07-28 14:23:26 +00004814 LocalDeclMap.erase(InitVD);
4815
4816 // Return the address of the private variable.
4817 return GetAddrOfLocalVar(PvtVD);
4818 });
4819 assert(IsRegistered && "firstprivate var already registered as private");
4820 // Silence the warning about unused variable.
4821 (void)IsRegistered;
4822
4823 ++OrigVarIt;
4824 ++InitIt;
4825 }
4826}
4827
Michael Wong65f367f2015-07-21 13:44:28 +00004828// Generate the instructions for '#pragma omp target data' directive.
4829void CodeGenFunction::EmitOMPTargetDataDirective(
4830 const OMPTargetDataDirective &S) {
Samuel Antaocc10b852016-07-28 14:23:26 +00004831 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4832
4833 // Create a pre/post action to signal the privatization of the device pointer.
4834 // This action can be replaced by the OpenMP runtime code generation to
4835 // deactivate privatization.
4836 bool PrivatizeDevicePointers = false;
4837 class DevicePointerPrivActionTy : public PrePostActionTy {
4838 bool &PrivatizeDevicePointers;
4839
4840 public:
4841 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4842 : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4843 void Enter(CodeGenFunction &CGF) override {
4844 PrivatizeDevicePointers = true;
4845 }
Samuel Antaodf158d52016-04-27 22:58:19 +00004846 };
Samuel Antaocc10b852016-07-28 14:23:26 +00004847 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4848
4849 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
Alexey Bataev475a7442018-01-12 19:39:11 +00004850 CodeGenFunction &CGF, PrePostActionTy &Action) {
Samuel Antaocc10b852016-07-28 14:23:26 +00004851 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev475a7442018-01-12 19:39:11 +00004852 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
Samuel Antaocc10b852016-07-28 14:23:26 +00004853 };
4854
Alexander Kornienko2a8c18d2018-04-06 15:14:32 +00004855 // Codegen that selects whether to generate the privatization code or not.
Samuel Antaocc10b852016-07-28 14:23:26 +00004856 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4857 &InnermostCodeGen](CodeGenFunction &CGF,
4858 PrePostActionTy &Action) {
4859 RegionCodeGenTy RCG(InnermostCodeGen);
4860 PrivatizeDevicePointers = false;
4861
4862 // Call the pre-action to change the status of PrivatizeDevicePointers if
4863 // needed.
4864 Action.Enter(CGF);
4865
4866 if (PrivatizeDevicePointers) {
4867 OMPPrivateScope PrivateScope(CGF);
4868 // Emit all instances of the use_device_ptr clause.
4869 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4870 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4871 Info.CaptureDeviceAddrMap);
4872 (void)PrivateScope.Privatize();
4873 RCG(CGF);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004874 } else {
Samuel Antaocc10b852016-07-28 14:23:26 +00004875 RCG(CGF);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004876 }
Samuel Antaocc10b852016-07-28 14:23:26 +00004877 };
4878
4879 // Forward the provided action to the privatization codegen.
4880 RegionCodeGenTy PrivRCG(PrivCodeGen);
4881 PrivRCG.setAction(Action);
4882
4883 // Notwithstanding the body of the region is emitted as inlined directive,
4884 // we don't use an inline scope as changes in the references inside the
4885 // region are expected to be visible outside, so we do not privative them.
4886 OMPLexicalScope Scope(CGF, S);
4887 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4888 PrivRCG);
4889 };
4890
4891 RegionCodeGenTy RCG(CodeGen);
Samuel Antaodf158d52016-04-27 22:58:19 +00004892
4893 // If we don't have target devices, don't bother emitting the data mapping
4894 // code.
4895 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
Samuel Antaocc10b852016-07-28 14:23:26 +00004896 RCG(*this);
Samuel Antaodf158d52016-04-27 22:58:19 +00004897 return;
4898 }
4899
4900 // Check if we have any if clause associated with the directive.
4901 const Expr *IfCond = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004902 if (const auto *C = S.getSingleClause<OMPIfClause>())
Samuel Antaodf158d52016-04-27 22:58:19 +00004903 IfCond = C->getCondition();
4904
4905 // Check if we have any device clause associated with the directive.
4906 const Expr *Device = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004907 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
Samuel Antaodf158d52016-04-27 22:58:19 +00004908 Device = C->getDevice();
4909
Samuel Antaocc10b852016-07-28 14:23:26 +00004910 // Set the action to signal privatization of device pointers.
4911 RCG.setAction(PrivAction);
4912
4913 // Emit region code.
4914 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4915 Info);
Michael Wong65f367f2015-07-21 13:44:28 +00004916}
Alexey Bataev49f6e782015-12-01 04:18:41 +00004917
Samuel Antaodf67fc42016-01-19 19:15:56 +00004918void CodeGenFunction::EmitOMPTargetEnterDataDirective(
4919 const OMPTargetEnterDataDirective &S) {
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00004920 // If we don't have target devices, don't bother emitting the data mapping
4921 // code.
4922 if (CGM.getLangOpts().OMPTargetTriples.empty())
4923 return;
4924
4925 // Check if we have any if clause associated with the directive.
4926 const Expr *IfCond = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004927 if (const auto *C = S.getSingleClause<OMPIfClause>())
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00004928 IfCond = C->getCondition();
4929
4930 // Check if we have any device clause associated with the directive.
4931 const Expr *Device = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004932 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
Samuel Antaobd0ae2e2016-04-27 23:07:29 +00004933 Device = C->getDevice();
4934
Alexey Bataev475a7442018-01-12 19:39:11 +00004935 OMPLexicalScope Scope(*this, S, OMPD_task);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00004936 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
Samuel Antaodf67fc42016-01-19 19:15:56 +00004937}
4938
Samuel Antao72590762016-01-19 20:04:50 +00004939void CodeGenFunction::EmitOMPTargetExitDataDirective(
4940 const OMPTargetExitDataDirective &S) {
Samuel Antao8dd66282016-04-27 23:14:30 +00004941 // If we don't have target devices, don't bother emitting the data mapping
4942 // code.
4943 if (CGM.getLangOpts().OMPTargetTriples.empty())
4944 return;
4945
4946 // Check if we have any if clause associated with the directive.
4947 const Expr *IfCond = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004948 if (const auto *C = S.getSingleClause<OMPIfClause>())
Samuel Antao8dd66282016-04-27 23:14:30 +00004949 IfCond = C->getCondition();
4950
4951 // Check if we have any device clause associated with the directive.
4952 const Expr *Device = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00004953 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
Samuel Antao8dd66282016-04-27 23:14:30 +00004954 Device = C->getDevice();
4955
Alexey Bataev475a7442018-01-12 19:39:11 +00004956 OMPLexicalScope Scope(*this, S, OMPD_task);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00004957 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
Samuel Antao72590762016-01-19 20:04:50 +00004958}
4959
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004960static void emitTargetParallelRegion(CodeGenFunction &CGF,
4961 const OMPTargetParallelDirective &S,
4962 PrePostActionTy &Action) {
4963 // Get the captured statement associated with the 'parallel' region.
Alexey Bataevddf3db92018-04-13 17:31:06 +00004964 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004965 Action.Enter(CGF);
Alexey Bataevc99042b2018-03-15 18:10:54 +00004966 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataev63cc8e92018-03-20 14:45:59 +00004967 Action.Enter(CGF);
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004968 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4969 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4970 CGF.EmitOMPPrivateClause(S, PrivateScope);
4971 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4972 (void)PrivateScope.Privatize();
Alexey Bataev60705422018-10-30 15:50:12 +00004973 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4974 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004975 // TODO: Add support for clauses.
4976 CGF.EmitStmt(CS->getCapturedStmt());
Arpith Chacko Jacob101e8fb2017-02-16 16:20:16 +00004977 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004978 };
Carlo Bertollib0ff0a62017-04-25 17:52:12 +00004979 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4980 emitEmptyBoundParameters);
Alexey Bataevddf3db92018-04-13 17:31:06 +00004981 emitPostUpdateForReductionClause(CGF, S,
4982 [](CodeGenFunction &) { return nullptr; });
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00004983}
4984
4985void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
4986 CodeGenModule &CGM, StringRef ParentName,
4987 const OMPTargetParallelDirective &S) {
4988 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4989 emitTargetParallelRegion(CGF, S, Action);
4990 };
4991 llvm::Function *Fn;
4992 llvm::Constant *Addr;
4993 // Emit target region as a standalone region.
4994 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4995 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4996 assert(Fn && Addr && "Target device function emission failed.");
4997}
4998
Arpith Chacko Jacobe955b3d2016-01-26 18:48:41 +00004999void CodeGenFunction::EmitOMPTargetParallelDirective(
5000 const OMPTargetParallelDirective &S) {
Arpith Chacko Jacob19b911c2017-01-18 18:18:53 +00005001 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5002 emitTargetParallelRegion(CGF, S, Action);
5003 };
5004 emitCommonOMPTargetDirective(*this, S, CodeGen);
Arpith Chacko Jacobe955b3d2016-01-26 18:48:41 +00005005}
5006
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00005007static void emitTargetParallelForRegion(CodeGenFunction &CGF,
5008 const OMPTargetParallelForDirective &S,
5009 PrePostActionTy &Action) {
5010 Action.Enter(CGF);
5011 // Emit directive as a combined directive that consists of two implicit
5012 // directives: 'parallel' with 'for' directive.
Alexey Bataevc99042b2018-03-15 18:10:54 +00005013 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5014 Action.Enter(CGF);
Alexey Bataev2139ed62017-11-16 18:20:21 +00005015 CodeGenFunction::OMPCancelStackRAII CancelRegion(
5016 CGF, OMPD_target_parallel_for, S.hasCancel());
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00005017 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
5018 emitDispatchForLoopBounds);
5019 };
5020 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
5021 emitEmptyBoundParameters);
5022}
5023
5024void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
5025 CodeGenModule &CGM, StringRef ParentName,
5026 const OMPTargetParallelForDirective &S) {
5027 // Emit SPMD target parallel for region as a standalone region.
5028 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5029 emitTargetParallelForRegion(CGF, S, Action);
5030 };
5031 llvm::Function *Fn;
5032 llvm::Constant *Addr;
5033 // Emit target region as a standalone region.
5034 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
5035 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
5036 assert(Fn && Addr && "Target device function emission failed.");
5037}
5038
Arpith Chacko Jacob05bebb52016-02-03 15:46:42 +00005039void CodeGenFunction::EmitOMPTargetParallelForDirective(
5040 const OMPTargetParallelForDirective &S) {
Alexey Bataevfb0ebec2017-11-08 20:16:14 +00005041 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5042 emitTargetParallelForRegion(CGF, S, Action);
5043 };
5044 emitCommonOMPTargetDirective(*this, S, CodeGen);
Arpith Chacko Jacob05bebb52016-02-03 15:46:42 +00005045}
5046
Alexey Bataev5d7edca2017-11-09 17:32:15 +00005047static void
5048emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
5049 const OMPTargetParallelForSimdDirective &S,
5050 PrePostActionTy &Action) {
5051 Action.Enter(CGF);
5052 // Emit directive as a combined directive that consists of two implicit
5053 // directives: 'parallel' with 'for' directive.
Alexey Bataevc99042b2018-03-15 18:10:54 +00005054 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5055 Action.Enter(CGF);
Alexey Bataev5d7edca2017-11-09 17:32:15 +00005056 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
5057 emitDispatchForLoopBounds);
5058 };
5059 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
5060 emitEmptyBoundParameters);
5061}
5062
5063void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
5064 CodeGenModule &CGM, StringRef ParentName,
5065 const OMPTargetParallelForSimdDirective &S) {
5066 // Emit SPMD target parallel for region as a standalone region.
5067 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5068 emitTargetParallelForSimdRegion(CGF, S, Action);
5069 };
5070 llvm::Function *Fn;
5071 llvm::Constant *Addr;
5072 // Emit target region as a standalone region.
5073 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
5074 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
5075 assert(Fn && Addr && "Target device function emission failed.");
5076}
5077
5078void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
5079 const OMPTargetParallelForSimdDirective &S) {
5080 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5081 emitTargetParallelForSimdRegion(CGF, S, Action);
5082 };
5083 emitCommonOMPTargetDirective(*this, S, CodeGen);
5084}
5085
Alexey Bataev7292c292016-04-25 12:22:29 +00005086/// Emit a helper variable and return corresponding lvalue.
5087static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
5088 const ImplicitParamDecl *PVD,
5089 CodeGenFunction::OMPPrivateScope &Privates) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00005090 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
5091 Privates.addPrivate(VDecl,
5092 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
Alexey Bataev7292c292016-04-25 12:22:29 +00005093}
5094
5095void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
5096 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
5097 // Emit outlined function for task construct.
Alexey Bataev475a7442018-01-12 19:39:11 +00005098 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
Alexey Bataevddf3db92018-04-13 17:31:06 +00005099 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5100 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
Alexey Bataev7292c292016-04-25 12:22:29 +00005101 const Expr *IfCond = nullptr;
5102 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
5103 if (C->getNameModifier() == OMPD_unknown ||
5104 C->getNameModifier() == OMPD_taskloop) {
5105 IfCond = C->getCondition();
5106 break;
5107 }
5108 }
Alexey Bataev24b5bae2016-04-28 09:23:51 +00005109
5110 OMPTaskDataTy Data;
5111 // Check if taskloop must be emitted without taskgroup.
5112 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
Alexey Bataev7292c292016-04-25 12:22:29 +00005113 // TODO: Check if we should emit tied or untied task.
Alexey Bataev24b5bae2016-04-28 09:23:51 +00005114 Data.Tied = true;
5115 // Set scheduling for taskloop
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00005116 if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
5117 // grainsize clause
Alexey Bataev24b5bae2016-04-28 09:23:51 +00005118 Data.Schedule.setInt(/*IntVal=*/false);
5119 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00005120 } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
5121 // num_tasks clause
Alexey Bataev24b5bae2016-04-28 09:23:51 +00005122 Data.Schedule.setInt(/*IntVal=*/true);
5123 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
Alexey Bataev2b19a6f2016-04-28 09:15:06 +00005124 }
Alexey Bataev7292c292016-04-25 12:22:29 +00005125
5126 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
5127 // if (PreCond) {
5128 // for (IV in 0..LastIteration) BODY;
5129 // <Final counter/linear vars updates>;
5130 // }
5131 //
5132
5133 // Emit: if (PreCond) - begin.
5134 // If the condition constant folds and can be elided, avoid emitting the
5135 // whole loop.
5136 bool CondConstant;
5137 llvm::BasicBlock *ContBlock = nullptr;
5138 OMPLoopScope PreInitScope(CGF, S);
5139 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
5140 if (!CondConstant)
5141 return;
5142 } else {
Alexey Bataevddf3db92018-04-13 17:31:06 +00005143 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
Alexey Bataev7292c292016-04-25 12:22:29 +00005144 ContBlock = CGF.createBasicBlock("taskloop.if.end");
5145 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
5146 CGF.getProfileCount(&S));
5147 CGF.EmitBlock(ThenBlock);
5148 CGF.incrementProfileCounter(&S);
5149 }
5150
Alexey Bataev61205822019-12-04 09:50:21 -05005151 (void)CGF.EmitOMPLinearClauseInit(S);
Alexey Bataev1e73ef32016-04-28 12:14:51 +00005152
Alexey Bataev7292c292016-04-25 12:22:29 +00005153 OMPPrivateScope LoopScope(CGF);
5154 // Emit helper vars inits.
5155 enum { LowerBound = 5, UpperBound, Stride, LastIter };
5156 auto *I = CS->getCapturedDecl()->param_begin();
5157 auto *LBP = std::next(I, LowerBound);
5158 auto *UBP = std::next(I, UpperBound);
5159 auto *STP = std::next(I, Stride);
5160 auto *LIP = std::next(I, LastIter);
5161 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
5162 LoopScope);
5163 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
5164 LoopScope);
5165 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
5166 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
5167 LoopScope);
5168 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
Alexey Bataev14a388f2019-10-25 10:27:13 -04005169 CGF.EmitOMPLinearClause(S, LoopScope);
Alexey Bataevf93095a2016-05-05 08:46:22 +00005170 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
Alexey Bataev7292c292016-04-25 12:22:29 +00005171 (void)LoopScope.Privatize();
5172 // Emit the loop iteration variable.
5173 const Expr *IVExpr = S.getIterationVariable();
Alexey Bataevddf3db92018-04-13 17:31:06 +00005174 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
Alexey Bataev7292c292016-04-25 12:22:29 +00005175 CGF.EmitVarDecl(*IVDecl);
5176 CGF.EmitIgnoredExpr(S.getInit());
5177
5178 // Emit the iterations count variable.
5179 // If it is not a variable, Sema decided to calculate iterations count on
5180 // each iteration (e.g., it is foldable into a constant).
Alexey Bataevddf3db92018-04-13 17:31:06 +00005181 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
Alexey Bataev7292c292016-04-25 12:22:29 +00005182 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5183 // Emit calculation of the iterations count.
5184 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
5185 }
5186
Alexey Bataev61205822019-12-04 09:50:21 -05005187 {
5188 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
5189 emitCommonSimdLoop(
5190 CGF, S,
5191 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
5192 if (isOpenMPSimdDirective(S.getDirectiveKind()))
5193 CGF.EmitOMPSimdInit(S);
5194 },
5195 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
5196 CGF.EmitOMPInnerLoop(
5197 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
5198 [&S](CodeGenFunction &CGF) {
5199 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
5200 CGF.EmitStopPoint(&S);
5201 },
5202 [](CodeGenFunction &) {});
5203 });
5204 }
Alexey Bataev7292c292016-04-25 12:22:29 +00005205 // Emit: if (PreCond) - end.
5206 if (ContBlock) {
5207 CGF.EmitBranch(ContBlock);
5208 CGF.EmitBlock(ContBlock, true);
5209 }
Alexey Bataevf93095a2016-05-05 08:46:22 +00005210 // Emit final copy of the lastprivate variables if IsLastIter != 0.
5211 if (HasLastprivateClause) {
5212 CGF.EmitOMPLastprivateClauseFinal(
5213 S, isOpenMPSimdDirective(S.getDirectiveKind()),
5214 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
5215 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
Stephen Kellyf2ceec42018-08-09 21:08:08 +00005216 (*LIP)->getType(), S.getBeginLoc())));
Alexey Bataevf93095a2016-05-05 08:46:22 +00005217 }
Alexey Bataev14a388f2019-10-25 10:27:13 -04005218 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
5219 return CGF.Builder.CreateIsNotNull(
5220 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
5221 (*LIP)->getType(), S.getBeginLoc()));
5222 });
Alexey Bataev7292c292016-04-25 12:22:29 +00005223 };
Alexey Bataev24b5bae2016-04-28 09:23:51 +00005224 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
James Y Knight9871db02019-02-05 16:42:33 +00005225 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
Alexey Bataev24b5bae2016-04-28 09:23:51 +00005226 const OMPTaskDataTy &Data) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00005227 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
5228 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
Alexey Bataev7292c292016-04-25 12:22:29 +00005229 OMPLoopScope PreInitScope(CGF, S);
Stephen Kellyf2ceec42018-08-09 21:08:08 +00005230 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
Alexey Bataev24b5bae2016-04-28 09:23:51 +00005231 OutlinedFn, SharedsTy,
5232 CapturedStruct, IfCond, Data);
Alexey Bataev7292c292016-04-25 12:22:29 +00005233 };
5234 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
5235 CodeGen);
5236 };
Alexey Bataev475a7442018-01-12 19:39:11 +00005237 if (Data.Nogroup) {
5238 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
5239 } else {
Alexey Bataev33446032017-07-12 18:09:32 +00005240 CGM.getOpenMPRuntime().emitTaskgroupRegion(
5241 *this,
5242 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
5243 PrePostActionTy &Action) {
5244 Action.Enter(CGF);
Alexey Bataev475a7442018-01-12 19:39:11 +00005245 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
5246 Data);
Alexey Bataev33446032017-07-12 18:09:32 +00005247 },
Stephen Kellyf2ceec42018-08-09 21:08:08 +00005248 S.getBeginLoc());
Alexey Bataev33446032017-07-12 18:09:32 +00005249 }
Alexey Bataev7292c292016-04-25 12:22:29 +00005250}
5251
Alexey Bataev49f6e782015-12-01 04:18:41 +00005252void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
Alexey Bataev7292c292016-04-25 12:22:29 +00005253 EmitOMPTaskLoopBasedDirective(S);
Alexey Bataev49f6e782015-12-01 04:18:41 +00005254}
5255
Alexey Bataev0a6ed842015-12-03 09:40:15 +00005256void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
5257 const OMPTaskLoopSimdDirective &S) {
Alexey Bataev61205822019-12-04 09:50:21 -05005258 OMPLexicalScope Scope(*this, S);
Alexey Bataev1e73ef32016-04-28 12:14:51 +00005259 EmitOMPTaskLoopBasedDirective(S);
Alexey Bataev0a6ed842015-12-03 09:40:15 +00005260}
Samuel Antao686c70c2016-05-26 17:30:50 +00005261
Alexey Bataev60e51c42019-10-10 20:13:02 +00005262void CodeGenFunction::EmitOMPMasterTaskLoopDirective(
5263 const OMPMasterTaskLoopDirective &S) {
5264 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5265 Action.Enter(CGF);
5266 EmitOMPTaskLoopBasedDirective(S);
5267 };
5268 OMPLexicalScope Scope(*this, S, llvm::None, /*EmitPreInitStmt=*/false);
5269 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
5270}
5271
Alexey Bataevb8552ab2019-10-18 16:47:35 +00005272void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(
5273 const OMPMasterTaskLoopSimdDirective &S) {
5274 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5275 Action.Enter(CGF);
5276 EmitOMPTaskLoopBasedDirective(S);
5277 };
Alexey Bataev61205822019-12-04 09:50:21 -05005278 OMPLexicalScope Scope(*this, S);
Alexey Bataevb8552ab2019-10-18 16:47:35 +00005279 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
5280}
5281
Alexey Bataev5bbcead2019-10-14 17:17:41 +00005282void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(
5283 const OMPParallelMasterTaskLoopDirective &S) {
5284 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5285 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
5286 PrePostActionTy &Action) {
5287 Action.Enter(CGF);
5288 CGF.EmitOMPTaskLoopBasedDirective(S);
5289 };
5290 OMPLexicalScope Scope(CGF, S, llvm::None, /*EmitPreInitStmt=*/false);
5291 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5292 S.getBeginLoc());
5293 };
5294 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,
5295 emitEmptyBoundParameters);
5296}
5297
Alexey Bataev14a388f2019-10-25 10:27:13 -04005298void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(
5299 const OMPParallelMasterTaskLoopSimdDirective &S) {
5300 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5301 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
5302 PrePostActionTy &Action) {
5303 Action.Enter(CGF);
5304 CGF.EmitOMPTaskLoopBasedDirective(S);
5305 };
5306 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
5307 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5308 S.getBeginLoc());
5309 };
5310 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,
5311 emitEmptyBoundParameters);
5312}
5313
Samuel Antao686c70c2016-05-26 17:30:50 +00005314// Generate the instructions for '#pragma omp target update' directive.
5315void CodeGenFunction::EmitOMPTargetUpdateDirective(
5316 const OMPTargetUpdateDirective &S) {
Samuel Antao8d2d7302016-05-26 18:30:22 +00005317 // If we don't have target devices, don't bother emitting the data mapping
5318 // code.
5319 if (CGM.getLangOpts().OMPTargetTriples.empty())
5320 return;
5321
5322 // Check if we have any if clause associated with the directive.
5323 const Expr *IfCond = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00005324 if (const auto *C = S.getSingleClause<OMPIfClause>())
Samuel Antao8d2d7302016-05-26 18:30:22 +00005325 IfCond = C->getCondition();
5326
5327 // Check if we have any device clause associated with the directive.
5328 const Expr *Device = nullptr;
Alexey Bataevddf3db92018-04-13 17:31:06 +00005329 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
Samuel Antao8d2d7302016-05-26 18:30:22 +00005330 Device = C->getDevice();
5331
Alexey Bataev475a7442018-01-12 19:39:11 +00005332 OMPLexicalScope Scope(*this, S, OMPD_task);
Alexey Bataevd2202ca2017-12-27 17:58:32 +00005333 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
Samuel Antao686c70c2016-05-26 17:30:50 +00005334}
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00005335
5336void CodeGenFunction::EmitSimpleOMPExecutableDirective(
5337 const OMPExecutableDirective &D) {
5338 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5339 return;
Akira Hatanaka9f37c0e2019-12-03 13:07:22 -08005340 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00005341 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
5342 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5343 } else {
Alexey Bataev6ab5bb12018-10-29 15:01:58 +00005344 OMPPrivateScope LoopGlobals(CGF);
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00005345 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
Alexey Bataevddf3db92018-04-13 17:31:06 +00005346 for (const Expr *E : LD->counters()) {
Alexey Bataev6ab5bb12018-10-29 15:01:58 +00005347 const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5348 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5349 LValue GlobLVal = CGF.EmitLValue(E);
5350 LoopGlobals.addPrivate(
Akira Hatanakaf139ae32019-12-03 15:17:01 -08005351 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); });
Alexey Bataev6ab5bb12018-10-29 15:01:58 +00005352 }
Bjorn Pettersson6c2d83b2018-10-30 08:49:26 +00005353 if (isa<OMPCapturedExprDecl>(VD)) {
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00005354 // Emit only those that were not explicitly referenced in clauses.
5355 if (!CGF.LocalDeclMap.count(VD))
5356 CGF.EmitVarDecl(*VD);
5357 }
5358 }
Alexey Bataevf138fda2018-08-13 19:04:24 +00005359 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5360 if (!C->getNumForLoops())
5361 continue;
5362 for (unsigned I = LD->getCollapsedNumber(),
5363 E = C->getLoopNumIterations().size();
5364 I < E; ++I) {
5365 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
Mike Rice0ed46662018-09-20 17:19:41 +00005366 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
Alexey Bataevf138fda2018-08-13 19:04:24 +00005367 // Emit only those that were not explicitly referenced in clauses.
5368 if (!CGF.LocalDeclMap.count(VD))
5369 CGF.EmitVarDecl(*VD);
5370 }
5371 }
5372 }
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00005373 }
Alexey Bataev6ab5bb12018-10-29 15:01:58 +00005374 LoopGlobals.Privatize();
Alexey Bataev475a7442018-01-12 19:39:11 +00005375 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
Alexey Bataeva8a9153a2017-12-29 18:07:07 +00005376 }
5377 };
5378 OMPSimdLexicalScope Scope(*this, D);
5379 CGM.getOpenMPRuntime().emitInlinedDirective(
5380 *this,
5381 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5382 : D.getDirectiveKind(),
5383 CodeGen);
5384}